{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 300)\n",
    "        self.fc2 = nn.Linear(300, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=1000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net1 = Net()\n",
    "criterion1 = nn.CrossEntropyLoss()\n",
    "optimizer1 = optim.SGD(net1.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores1, train_scores1, test_scores1 = fit(net1, criterion1, optimizer1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1fnH8c+TnWwQSAiQENYAsqhAAPdWQMSq4FqFatXa2v4qXdRq9ddW/dnW2loX2lorVau2Vqq4IYIoCKK4lACyEwhrFkgCSQgJZJnM8/vjjhrChAwwyWRmnvfrlRe5957MPJcbvq/LmXPPEVXFGGNM8IsIdAHGGGP8wwLdGGNChAW6McaECAt0Y4wJERboxhgTIqIC9capqanat2/fQL29McYEpZUrV+5T1TRvxwIW6H379iU3NzdQb2+MMUFJRHa1dMy6XIwxJkRYoBtjTIiwQDfGmBBhgW6MMSHCAt0YY0KEBboxxoQIC3RjjAkRARuHbowxIUsVilZCfQ0ADY3K3qpaiioOU1R5mKEjRnPK4MF+f1sLdGOMaUJVEZFW27ndSlHlYXbur2F/dT37a+o5cLiBKNchJuX/hiH73/uybTTQ2/MF8Jn7lzD4Tr/XboFujOmQvgjMvVW1JMdF0zUhhi7x0UT6ELZfUKDqcAP7a+qpOFRPg8vttc2OfTX8d0c5/91RTnlNPSkJ0aTEx5AUF4Vw9PtV17nYvq+a2oYjX68X+/h7zCMMkt082ng1u5NGkpnSiYyUTvROiadXl05kdOnEuO4Dj/evwycW6MaYk1dXDe/9Cko3n/BL1De6OVjbwMFaF9V1LmobGmn0LKhW6fk6WZEt7M8GhkUKP46LJqZrBK5GN67DSuMh7yu6RYgQ1zmSTtERxEVHEh0ZQXSkELk/D3G74cqXuX3QJD9UfHws0I0xJ6diJ/rSNCjbTFXaGPByBy0iX4aeW+FwfSOHG1wcqm/0fN9IvefuOTJCSIyLolunOOJjIomJiqDRrbga3TQ0Hv+SmZERX713S10pMZERxEVHeLkXP059zoaJ90Nq9sm+0gmxQDfGtG7fVqgqPmKXohQU7CL1w1/R6HLxw/q7+HD3qcf1somxUQzsnsjA7okM6ZHEmL5dGdYrmahIG4B3IizQjTHHtvMjeO4SnN7mrwiQBeRrBk/2/A2TR45mRlqi15c43NBIeU095TX1RIiQnZ5Idvck0pNjffoA0vjGAt0Y0zJXPcy7He3Sm/VjHuLVlUVs3FNFVGQEp2V2Zmy/bow84+s80rlLoCs1WKAbE362vAuf/BkaXc52XDJc8ACkOeOiVZWyg3VsLa0m7tOZjN6Xx/8l3cdzb0F6cn++f/EArs7JJCkuOoAnYbyxQDcmXKjCh4/A+7+BlD7Q2RkV7S5Ygfup83mt3/38p2o4W0sOUlXrIlNKeS/mbyySsayLP4NfX5bB1aMziYtuaayICTQLdGPCQX0NvHkrbHgdRlxN4yUzWbajhhc/28Wmyk08Gf0oV225C02cTs2AMWR0iWds8avE7otiwq3PMrFL79bfwwScT4EuIpOBmTjDOJ9W1YeaHc8Cnge6eNrcrarz/VyrMeZEVO6G2dNh73qqz/0Vz8tU/v3YZxRVHiY1MZarzhvL4f5vo2vv5ZoNL8K2F7/62QsfBAvzoCGqxx7XKSKRwBbgAqAQWAFMU9WNTdrMAlar6pMiMhSYr6p9j/W6OTk5amuKGtPGdi6Hl6+n0dXAU6n38OjOvrjcypn9u3HdGX24YGg6MVGeIYKqsGcNNBxytmMSoOdpgavdeCUiK1U1x9sxX+7QxwL5qrrd82KzganAxiZtFEj2fN8ZOHLAqjGm/a14BhbcRU1Cb75Z8xMKintx41m9mTYuiwHehheKQK/T279O4ze+BHoGUNBkuxAY16zN/cC7IvIjIAGY6O2FROQW4BaArKys463VGOMLVz0suAtW/oOdXc9myp6byOrVg3euz6FXl06Brs60IV8C3duo/+b9NNOA51T1ERE5E/iniAxX1SNmrlHVWcAscLpcTqRgY0wzNfuhZJ3zvbppXPJ7Igs/5ZVO3+TnxVO49PRMHrriVDrF2OiUUOdLoBfy1ayPAJkc3aVyMzAZQFU/EZE4IBUo9UeRxhjvDuYtJfrVm4irL/9yXz0x3Fn/I/KSL+DBK/pxzZje9jRmmPAl0FcA2SLSDygCrgWmN2uzG5gAPCcipwBxQJk/CzXGOFSVBev3UvL+X7mu/Al2a3ce0nuI7pRI505RxHXvz7fPymFM3xQL8jDTaqCrqktEZgALcYYkPquqG0TkASBXVecCdwB/F5HbcLpjbtTWhs8YY3xXfwgW3Ud94Rp27q8h83AN34jYwbaUszh86d+Y1T/Lwtv4Ng7dM6Z8frN99zb5fiNwtn9LM8ZsK6tm9/Y8Tlv+Q1Kq8lirQ2ggkj7d0nCfNp0B590BEdY3bhz2pKgxHVBJVS0PL8xj9+pF/DX6caJo4DsNP8M14AJ+e9kIenWLD3SJpgOyQDemA6ltaOTvy7bz5AfbSG0s4724P0JSDw5e/gJPZQ776iEgY7ywQDemA1BV5q4p5vcLNlN8oJbJw3rwKM8Tu0vhxteJTekb6BJNELBANyaAVJVlW/fx+KItrN5dybBeyTx6zemc4cqFf8+HCfeChbnxkQW6MQHy/uYSZi7ayprCA/TqHMcfrjyVK0dnEuk6DH/9GaQOhjN/FOgyTRCxQDemnbndyu/f2cxTy7bTu2snfnfFCK4clflV//iyh50ZEm+cD1ExgS3WBBULdGPaUb3LzV1z1vDG58Vcd0YW9106jOjIJrMdfvSY83XadOhrI4HN8bFAN6adlFbVcscra/hw6z5+NmkQt54/8KuHgeoPwdwZsP5VGHYFXPxIYIs1QckC3Zg2VtvQyDMf7eCJJfk0NLr5w1Wn8s2cJtMjVRbAf74Fe9Y6H4Kec7szla0xx8kC3Zg2oKpsKalm/ro9zFlZSFHlYS4cls49F51C39SErxru+hj+cz246mDabBg8OXBFm6BngW6Mn727YS8PvbOZ7WU1iMDYvl15+OpTOWtA6pENc5+F+XdClz4w7SVIGxyYgk3IsEA3xk9qGxr53fxNPP/JLob0SOI3lw1n0rB0uifFHd1463sw7zYYOBGufAY6dWn/gk3IsUA3xg927Kvhhy+uYtOeKm4+px93TR5MbFQLk2bVH4K373DGmV/7kg1NNH5jgW7MSVqaV8qPXlpNVITw7I05jB+Sfuwf+PARqNwFN8yzMDd+ZYFuzAlSVWYt287v39nM4B7JzLp+NL27tjILYlkeLJ8Jp02Dfue2T6EmbFigG3MCtpQc5NfzNvLh1n1cPKInD199KvExnn9OjQ1Q8Bk01h/9g8v+CDEJcMGv27dgExZ8CnQRmQzMxFmx6GlVfajZ8ceA8z2b8UB3VbVPeUzIqaip57FFW3jxs93Ex0Ry36VDufGsvl89IFSzD17+Nuxa3vKLXDoTEtPap2ATVloNdBGJBJ4ALsBZMHqFiMz1rFIEgKre1qT9j4CRbVCrMQGjqsxbu4f7526g8nAD3xqXxU8nDqJrQpM+8D1rYfZ0qCmDix+F9GFHv1BsMqQPbb/CTVjx5Q59LJCvqtsBRGQ2MBXY2EL7acB9/inPmABzu6le/Hv25r5Fz8MN/Csmkj69E4jfFwmzm7Xdsxbiu8JNCyBjVEDKNeHNl0DPAAqabBcC47w1FJE+QD/g/RaO3wLcApCVlXVchRrT3rT2AMXPfpuM0qVU60B6dOtGry6diGjpqfyhU5y+8aRWRrkY00Z8CXRvv77aQttrgTmq2ujtoKrOAmYB5OTktPQaxgRc0bZ18NJ00hsKebbzDxl//S/ITEsMdFnGHJMvgV4INJlJiEyguIW21wK3nmxRxgTKwdoGFrzxIhdu+l/cRLBk3N+5cfKVRLR4W25Mx+FLoK8AskWkH1CEE9rTmzcSkcFACvCJXys0ph2UVNUyb00xVUse58eNL1AS15/ob73EBVk2v4oJHq0Guqq6RGQGsBBn2OKzqrpBRB4AclV1rqfpNGC2qlpXigkKlTV1FLx8J9HF/6WmzsVZ1HFKxG4q+32DXtOfdsaLGxNEJFD5m5OTo7m5uQF5bxPe1hZW8vzHu2hcN4fHI//E5sjBxCd1ITUxlvihk+DMGTYfuemwRGSlquZ4O2ZPipqwUVhxiIcWbGbe2j2kx9SxKPZFDqecypD/WQoRLUykZUwQsUA3Ia22oZFVuytYsrmU5z/ZRYTAjydkM6N2FjEry+GyORbmJmRYoJuQ9Nn2/Ty+aCu5u8ppaFQiBC49rRc/nzyEXoc2w9+fgTHftQeATEixQDchZff+Q/xuwSYWrN9Lz85xfOfsfozr35Wcvl1JjouGmv3w5o8gIQ0m/CrQ5RrjVxboJiSoKs9/vJMHF2wmUoQ7LhjEd8/tT6eYJt0pe9c5c60cLIFr/glxnQNXsDFtwALdBL3ymnrumrOGRZtKmTCkOw9eMYL05GbLvm18E17/gRPi31kAGaMDU6wxbcgC3QStA4caeHVVIU8t20ZFTcPRU9l+oWIXvHKT019+zb8gqUdgCjamjVmgm6Bx4FAD+WUH2VpSzX93lvP22j3UudyMzOrCMzeMYXhGC10oK58DFK76h4W5CWkW6KbDqzxUz6/nbeLVVYVf7kuMjeKq0ZlMH5fFsF7H6At31cPqf0L2hdCld8vtjAkBFuimQ5u/bg/3vrmeykMNfPecfpw1sBsD05LISOlEpC8TZm1+y1lwYszNbV+sMQFmgW46JFXlt29v4umPdjAiozMvfGccQ3slH/8LrXgWuvSBARP8X6QxHYwFuumQHlu0lac/2sENZ/bhV5cMJSoy4vhfpHQz7PoIJt4PESfw88YEGQt00+HMWraNPy3eyjU5vbl/yrCjR634KvdZiIyBkdf7t0BjOigLdNNhNDS6+euSbTy2aAsXn9qTB68YceJhvn8brJkNQ6dCQqp/CzWmg7JANx3C+qID3DVnLRv3VDH19F48fNVpvn3o6U3+Yphzk9PNcs7t/i3UmA7MAt0E1IbiA/zr0128nFtI14QY/nbdaCYPP46x4qpQtBIOVzrbxath6YOQdgpM+zek9G2Tuo3piHwKdBGZDMzEWbHoaVV9yEubbwL34ywgvUZVj1qmzpgvLM0r5fFFW/m8oJLYqAiuHdObuy4cQuf46ON7odxn4e1md+GnTIHLnoRYW9TZhJdWA11EIoEngAtwFoxeISJzVXVjkzbZwD3A2apaISLd26pgE/z++eku7ntzPVld4/nVJUO5alTm8Qc5OJNsLfo/6HsuTLjP2RcVCz1G2IpDJiz5coc+FshX1e0AIjIbmApsbNLme8ATqloBoKql/i7UBD9V5Y/v5vHEkm1MGNKdP08fSXzMSfT6vftLcB2GSx6H1IH+K9SYIOXLv6YMoKDJdiEwrlmbQQAishynW+Z+VX2n+QuJyC3ALQBZWVknUq8JMvmlB1mxs4L80mrWFFSSu6uCa3J689vLh5/Y2PIvbF8K616Gr/3cwtwYD18C3dv/XZuvLB0FZANfBzKBD0VkuKpWHvFDqrOAWeAsEn3c1ZoOq9HtrAr0xTDDtYWV/GlxPos2lQAQFx3BgLRE7rloCLec1//4hyO66mD3J9DoAhTeuRtS+tkoFmOa8CXQC4GmsxplAsVe2nyqqg3ADhHJwwn4FX6p0nRI1XUu3t9cyjvr97Bkcxkut5uU+BgS46LYXlZDclwUP52YzRUjM8lM6UTEiQ5DrCqG/1znjGb5ksB1cyA6rsUfMybc+BLoK4BsEekHFAHXAs1HsLwBTAOeE5FUnC6Y7f4s1HQsb6wu4pdvrKe6zkVaUiyXj8ogOS6a8po6Kg41cNXoTK4/ow9JcSfwYWdTBSucMK+vdkaudMt29id0g679T/5EjAkhrQa6qrpEZAawEKd//FlV3SAiDwC5qjrXc2ySiGwEGoE7VXV/WxZuAqOmzsWv3lzPa6uKGNM3hTsvHMLoPikn/hCQ2w3LH4fNb3s/vnctJPeC61+H9KEnXrgxYUBUA9OVnZOTo7m5uQF5b+M7V6Ob5dv2k7e3iq0l1Xy8bT97Dhxmxvhsfjx+4Ml9sFl30FkWbvM8yMjxvsZnci+44AGI73ri72NMCBGRlaqa4+2YPSlqvKp3uXljdRFPLM1n1/5DAKQlxTIoPZE/Xn0aZw7odnJvUL4DXpoG+/Lgwt/BGf9jY8eNOUkW6AaAdYUHmLe2mLKDdZQfqidv70H2HKhlREZnnvzWKM4akHpiD/9446qHf18D1SVw3Wsw4Hz/vK4xYc4CPcyt3FXBX97fypK8MmIiI0hLiqVrQgzDMzrz4BUj+PqgtBOf8bAln/zZuTOf/oqFuTF+ZIEeBlSV0oN1bC2pZmvpQfJLq9laWk1+aTXlNfWkxEdz54WDuf7MPiSf7KiU1pTvgA/+4My3MmhS276XMWHGAj3EbS+r5p7X1vHZjvIv93XuFE1290QmDU1neEZnLh+ZQUJsO/wqqMKCuyAiCiYfNb+bMeYkWaCHKFejm6c/2sFj720hJiqCn08ewmm9OzOweyJpibH+70bxxaa3YOu7cOGD0Dmj/d/fmBBngR5iXI1u5q3dw1+W5JNfWs2koen8+rLhpCcH+InKTW85QxTTR8DY7we2FmNClAV6iCitqmXB+r38Y/kOdu4/xOD0JP523SguHNYjMHfjX3C7YdkfYOnvIGM0XPMviLRfO2Pagv3L6uBKq2pZuGEvizaVUnm44cv9CTGRpCTE0DU+hk17qli5uwJVGJ6RzN+uG82koeknPneKv6jCG/8Da2fDadPhksds7hVj2pAFegekqizdUsZTH2zjsx3lqEL/1AR6d413juM8gr+xuIrymnp6denEbRMHcdHwHmSnJwW2+KY2z3PC/Lw74fxf2INDxrQxC/QOpN7l5v3NJTyxZBvrig6Q0aUTP50wiItG9CC7e2Jgu06OV101LPg5pA+Hr91tYW5MO7BAD7CGRjdL88qYv24PizaVcLDWRZ9u8fzhylO5bGQGMVEnMVdKIC39HVQVwdXPWZ+5Me3E/qW1k3qXm5W7KoiPiaRrQgyNbuXVVYXMXlFA2cE6OneKZvKwHlw0ogfnZaed3KRXgbZ3PXz6JIy6AXqPDXQ1xoQNC/R2sKagkrvmrCWv5OAR+0Vg/ODuTB+XxXmD0ogO5hD/wqFyePNW6NQFJt4f6GqMCSsW6G2ops7FzMVbefrD7XRPiuPxa04nMTaK8pp6al2NjB/SncyU+ECX6T8lG2H2NGeFoaufsylvjWlnFuhtoKq2gRc+3skzH+2g4lAD08dlcfdFQ9p+npT2VrQKavY531cVwsJfQmwi3Dgfeo8JbG3GhCGfAl1EJgMzcVYselpVH2p2/EbgYZwl6gD+oqpP+7HOoNDoVv6xfAczF2/lYK2LCUO6M2P8QEZmpQS6NP9qdMF7v4JP/3rk/i8eHEruFZi6jAlzrQa6iEQCTwAX4CwGvUJE5qrqxmZN/6OqM9qgxqCwpeQgd81Zy+cFlZw/OI07Jg1meIaXFXiC3aFymHMTbF/qPMJ/6jXOfhHoMQIiQ+x/IcYEEV/u0McC+aq6HUBEZgNTgeaBHrZezi3gF6+vIykumj9NG8mlp/YMrjHjx1K4EhbdB/U1zvaBQqithCl/gVHXB7Y2Y8wRfAn0DKCgyXYhMM5LuytF5DxgC3CbqhY0byAitwC3AGRlZR1/tR3Qx9v28b+vreOM/t2Yee3pdEuMDXRJ/vP5S/DWTyC+G6QPc/Yl94Kzf2LDEY3pgHwJdG+3ms1Xln4LeElV60TkB8DzwPijfkh1FjALnEWij7PWDqeg/BC3vriKvqkJPHndKJKC4UPP2ioo+AzUfex2296Hz/4Gfc+Fq5+HhJNcQ9QY0+Z8CfRCoHeT7UyguGkDVd3fZPPvwO9PvrSOrbrOxXefz8Wt8PS3c4IjzEs3O8MKy7f71n7cD2DSb6xf3Jgg4UugrwCyRaQfziiWa4HpTRuISE9V3ePZnAJs8muVHdA9r60jv6yaF74zlr6pCYEup3Wb58Nr34PoeLj235DU49jtY5MhNbt9ajPG+EWrga6qLhGZASzEGbb4rKpuEJEHgFxVnQv8WESmAC6gHLixDWsOuLfWFPPWmmJ+NmkQZw9MDXQ5rVv5vNMX3ut0uOZFWy3ImBAlqoHpys7JydHc3NyAvPfJKK2qZdLjy+iXmsAr3z+z48+5cqAQ/jLWedBn2myI7hToiowxJ0FEVqpqjrdjHTyNOhZV5eevrqW2oZFHrj6t44c5OFPYqhsu/ZOFuTEhLggSqeN46b8FLMkr4+7JQ+iflhjoclqX946zyMTX7oKUPoGuxhjTxizQfbRoYwn3vrmec7NT+faZfQNdTuvqD8GCOyFtCJwZtg/wGhNWbHIuH3ycv48f/nsVQ3sl89dvjQr8Wp2tcTfCOz+Hyt3ORFlRMYGuyBjTDizQW7FqdwXffSGXft0SeP6msR1/vPnhCpjzHefBoLN/Cn3PDnRFxph2YoF+DA2Nbn780mrSkmL5581jSUno4He6Xzw4VFngfAg6+oZAV2SMaUcW6Mfw1ppiCisO88wNOXRPjgt0OceWtwBe/Z4zkuXGeZB1RqArMsa0Mwv0Frjdyl+XbmNIjyTGD+ke6HJapgof/hHe/y30PA2ufRE6Zwa6KmNMAFigt+C9TSXkl1Yz89rTAz8VbnUpzP8ZVOw6+lh9DezfCiO+CVNsrLkx4cwC3QtV5+48q2s8F4/oGdhiilfD7G85C0v0Oxevk1+O+z6M+a6zyIQxJmxZoHvxybb9rCmo5DeXDW//p0H3rIEqzzxnlbvgvXshIQ1ufhd6ntq+tRhjgooFuhdPfrCNtKRYrhrdzn3RW9+DF686cl+fs+GbL0BCEEwCZowJKAv0ZooqD/Ph1n3cNnEQcdGR7ffGDYedfvJu2XDFU4BARCR0HwaRdpmMMa2zpGjmjdVFAFw+sp2nmP3wEajYCTe8BRmj2/e9jTEhweZyaUJVeX11ETl9UsjqFt9+b1y2BT56HE69Bvqd137va4wJKXaH3sSG4iryS6v57eXD/f/irnrY+SE01h99bPmfICbeWe7NGGNOkE+BLiKTgZk4KxY9raoPtdDuKuAVYIyqBt3qFa+tKiImMoJLRvTy7wtXl8LL34bdn7Tc5tKZkNiBH2AyxnR4rQa6iEQCTwAX4CwYvUJE5qrqxmbtkoAfA5+1RaFtzdXoZu6aYsYP6U7neD9OwFX8uWcc+X6Y8mfoMeLoNjFJkDrQf+9pjAlLvtyhjwXyVXU7gIjMBqYCG5u1+zXwB+Bnfq2wnXyUv4991XVc5uuHoSufh8IVcOGDEJfcZP9zzjFtdLbL8iA+Fb7zjrOmpzHGtBFfPhTNAAqabBd69n1JREYCvVV13rFeSERuEZFcEcktKys77mLb0uuri+jcKZrzh6S13nj/NmeI4ep/wtMTnW1XPbz1U2cxZrcLkno6XyOugluWWJgbY9qcL3fo3p4n/3JlaRGJAB4DbmzthVR1FjALnEWifSux7R2sbWDhhr1cOSqT2KhWxp6rwtt3QFQcXPYkzL8T/n4+dBsIRSudOcgn3OuMITfGmHbkS6AXAr2bbGcCxU22k4DhwFLPJFY9gLkiMiVYPhh9a80eahvcXJ3Tu/XGG16D7Uvgooedu+/MHKePvGQDXPmMs88YYwLAl0BfAWSLSD+gCLgWmP7FQVU9AHz5XLqILAV+FixhDvBybgGD0hM5LbPzsRvWHoB37oGep8OYm519KX3he+87x2yUijEmgFrtQ1dVFzADWAhsAl5W1Q0i8oCITGnrAtvalpKDfF5QyTdzeh97mlxXPcy73RmCeMljR3apRMVamBtjAs6nceiqOh+Y32zfvS20/frJl9V+Xl5RQFSEHPtR/+oyeOUG2LUczv8FZIxqvwKNMcZHYf2kaL3Lzeuri5h4SjrdEmO9N9qzFmZPh5oyuOLvcOo327dIY4zxUVgH+vubS9lfU881Y1r4MLS2Cl68GiQCblpgd+bGmA4trAP95dwC0pNjOTe7hbnGl/wWqkucDz0tzI0xHVzYzrZYWlXL0rxSrhyV6X1VouLP4b+znKXdLMyNMUEgbAP99dVFuBXvqxK5G2Hebc4j++N/2f7FGWPMCQjLLhdV5dVVhYzK6kL/tMSjG+Q+C8WrnAeFOnVp/wKNMeYEhOUd+rqiA2wpqeZKb3fnB0tg8a+h39dg+JXtX5wxxpygsAz0V1cWEhMVwSWnepn3/N1fgOswXPwoHOtBI2OM6WDCLtDrXI28uaaYSUPT6dyp2bzn25fCulfgnNtsfnJjTNAJu0BfsrmUykMNR38Y6qpzZlFM6Qfn3B6Y4owx5iSE3Yeic1YW0T0plnOzm817vnwm7M+H616D6LjAFGeMMSchrO7QKw/VszSvlMtHZhAZ0aR/vLHBCfRTpsDACYEr0BhjTkJYBfqHW/fhcisXDu9x5IHi1VBfbaNajDFBLawC/YMtZXTuFM1pmc3Glu/8yPmzz9ntX5QxxvhJ2AS6qrJsSxnnZKce2d0CzrS4qYMh0Yf1RI0xpoMKm0DfvPcgpQfr+NqgZqHd6ILdn0Jfuzs3xgQ3nwJdRCaLSJ6I5IvI3V6O/0BE1onI5yLykYgM9X+pJ2fZljKAowN97xqn/9y6W4wxQa7VQBeRSOAJ4CJgKDDNS2D/W1VHqOrpwB+AR/1e6Un6YEsZQ3okkZ7cbEjizuXOn33Paf+ijDHGj3y5Qx8L5KvqdlWtB2YDU5s2UNWqJpsJgPqvxJNXU+dixc7yo+/Owek/7zoAknocfcwYY4KILw8WZQAFTbYLgXHNG4nIrcDtQAww3tsLicgtwC0AWVlZx1vrCft0+34aGpXzmge6uxF2fQLDpnr/QWOMCSK+3KF7m6HqqDtwVX1CVQcAPwe8TiKuqrNUNUdVc9LS2m9EyQdbyugUHUlO35QjD5Ssh7oD0Me6W4wxwc+XQC8Emi66mQkUH6P9bOCykynK35ZtKePMAd2IjYo88sCX/ef2gagxJvj5EugrgJuNnPUAAAvKSURBVGwR6SciMcC1wNymDUQku8nmxcBW/5V4cnbuq2Hn/kOc523d0F3LoUsf6OxlXnRjjAkyrfahq6pLRGYAC4FI4FlV3SAiDwC5qjoXmCEiE4EGoAK4oS2LPh6LN5cCMH5I+pEH3G7Y9TEMvigAVRljjP/5NNuiqs4H5jfbd2+T73/i57r8ZvGmErK7J5LVLf7IA2Wb4HC5jT83xoSMkH5StKq2gf/uKGfCKelHH7T+c2NMiAnpQP8grwyXW5l4SvejD+76CJIznT50Y4wJASEd6O9vLiUlPpqRWc2GK6o6/ed9z7Z1Q40xISNkA93V6GZJXinnD+5+9OyK+7ZATZn1nxtjQkrIBvqq3ZVUHmpoof/cM/+5zd9ijAkhIRvoizeVEB0pnDeohfHniT2ga//2L8wYY9pIyAb6ok0ljOvXjaS46CMPqDojXKz/3BgTYkIy0Hfuq2FbWQ0TvI1uKd8O1Xut/9wYE3JCMtDf21gCwETrPzfGhJGQDPSFG/YytGcyvbvGH31w13JISIPUQe1fmDHGtKGQC/Syg3Ws3F3BpGFe7s6/6D/vc5b1nxtjQk7IBfqiTSWowoXDvKxAtOMDqCqEfl9r/8KMMaaNhVygL9ywl6yu8QzpkXTkAVcdvH0HpPSD078VmOKMMaYNhVSgH6xt4OP8/Vw4LB1p3qWyfCbsz4eLH4HoOO8vYIwxQSykAn1JXhn1jW4mNe9u2b8Nlv0Rhl0OAycEpjhjjGljIRXo727YS2piDKOaTsalCvPvhMgYuPB3gSvOGGPamE+BLiKTRSRPRPJF5G4vx28XkY0islZEFotIu89JW+dqZGleGRcMTT9yMq6dH8G2xXD+/0Jyz/Yuyxhj2k2rgS4ikcATwEXAUGCaiAxt1mw1kKOqpwJzgD/4u9DWfLR1H9V1rqO7W3KfgbgukHNTe5dkjDHtypc79LFAvqpuV9V6YDYwtWkDVV2iqoc8m58C7b7q8muriuiaEMPZA5pMxnWwBDa95Yxqie7U3iUZY0y78iXQM4CCJtuFnn0tuRlY4O2AiNwiIrkikltWVuZ7la2oPFTPextLmHp6L2KimpzS6hfA7YKc7/jtvYwxpqPyJdC9PVKpXhuKXAfkAA97O66qs1Q1R1Vz0tLSfK+yFW+t3UN9o5srRzX5j4G7EXKfcx4iSh3ot/cyxpiOypdALwR6N9nOBIqbNxKRicAvgCmqWuef8nwzZ2UhQ3okMaxX8lc7tyx0ngodc3N7lmKMMQHjS6CvALJFpJ+IxADXAnObNhCRkcBTOGFe6v8yW5ZfepA1BZVcNTrzyIeJcp9xFrEY/I32LMcYYwKm1UBXVRcwA1gIbAJeVtUNIvKAiEzxNHsYSAReEZHPRWRuCy/nd3NWFhEZIUw9vUm3/o4PIX8xjL4BIqNb/mFjjAkhUb40UtX5wPxm++5t8v1EP9flk0a38vrqQr4+KI20pFhn54pnYMFd0G0AjPleIMoyxpiACOonRT/eto+SqjquGp0JjS5466fw9u3Q/3z47mJI9N8Hr8YY09H5dIfeUS3csJf4mEjOH9Idti6Elf+AM2fABQ9ARGSgyzPGmHYVtHfoqsqSzWWcPTCVuOhI2JfnHPj63RbmxpiwFLSBvqWkmqLKw4wf4lkIunwHxKdCbNKxf9AYY0JU0Ab6+5ud0ZHnD/YEesUO6NovgBUZY0xgBW2gL9lcytCeyfTo7FmsonynsxqRMcaEqaAM9AOHGli5u+Kr7hZXvfNUqN2hG2PCWFAG+gdby2h0qzO6BaByN6jb7tCNMWEtKAN9yeZSuibEcHrvLs6Oih3On3aHbowJY0EX6I1uZWleKV8blPbVykTlnkC3O3RjTBgLukBfU1hJxaGGr7pbwLlDj06AxO4t/6AxxoS4oAv0pXllREYIX8tu8lh/+Q5I6Qvibep2Y4wJD0H36P+t5w9g/JDudI5vMotixQ7oZotYGGPCW9DdocdGRX71YSiA2w0VO507dGOMCWNBF+hHqd4Lrlob4WKMCXvBH+g2wsUYYwAfA11EJotInojki8jdXo6fJyKrRMQlIlf5v8xjsDHoxhgD+BDoIhIJPAFcBAwFponI0GbNdgM3Av/2d4GtKt8BEgmde7fe1hhjQpgvo1zGAvmquh1ARGYDU4GNXzRQ1Z2eY+42qPHYKnZAl962dqgxJuz50uWSARQ02S707DtuInKLiOSKSG5ZWdmJvMTRyndY/7kxxuBboHt7WkdP5M1UdZaq5qhqTlqan9b7tHnQjTEG8C3QC4GmHdSZQHHblHOcDlfC4Qq7QzfGGHwL9BVAtoj0E5EY4FpgbtuW5SMb4WKMMV9qNdBV1QXMABYCm4CXVXWDiDwgIlMARGSMiBQCVwNPiciGtiz6S2WehaHtDt0YY3yby0VV5wPzm+27t8n3K3C6YtqPqw6WPeyEedrgdn1rY4zpiIJucq4vLZ8J+/PhuldtyKIxxhCsj/7v3wbL/gjDLoeBEwNdjTHGdAjBF+iqMP9OiIyBC38X6GqMMabDCL5A3/gGbFsM438JyT0DXY0xxnQYwRfoMUkw+GIY891AV2KMMR1K8H0omj3R+TLGGHOE4LtDN8YY45UFujHGhAgLdGOMCREW6MYYEyIs0I0xJkRYoBtjTIiwQDfGmBBhgW6MMSFCVE9oNbmTf2ORMmDXCf54KrDPj+UEi3A873A8ZwjP8w7Hc4bjP+8+qup1Dc+ABfrJEJFcVc0JdB3tLRzPOxzPGcLzvMPxnMG/521dLsYYEyIs0I0xJkQEa6DPCnQBARKO5x2O5wzhed7heM7gx/MOyj50Y4wxRwvWO3RjjDHNWKAbY0yICLpAF5HJIpInIvkicneg62kLItJbRJaIyCYR2SAiP/Hs7yoi74nIVs+fKYGu1d9EJFJEVovIPM92PxH5zHPO/xGRmEDX6G8i0kVE5ojIZs81PzNMrvVtnt/v9SLykojEhdr1FpFnRaRURNY32ef12orjT55sWysio473/YIq0EUkEngCuAgYCkwTkaGBrapNuIA7VPUU4AzgVs953g0sVtVsYLFnO9T8BNjUZPv3wGOec64Abg5IVW1rJvCOqg4BTsM5/5C+1iKSAfwYyFHV4UAkcC2hd72fAyY329fStb0IyPZ83QI8ebxvFlSBDowF8lV1u6rWA7OBqQGuye9UdY+qrvJ8fxDnH3gGzrk+72n2PHBZYCpsGyKSCVwMPO3ZFmA8MMfTJBTPORk4D3gGQFXrVbWSEL/WHlFAJxGJAuKBPYTY9VbVZUB5s90tXdupwAvq+BToIiI9j+f9gi3QM4CCJtuFnn0hS0T6AiOBz4B0Vd0DTugD3QNXWZt4HLgLcHu2uwGVqurybIfi9e4PlAH/8HQ1PS0iCYT4tVbVIuCPwG6cID8ArCT0rze0fG1POt+CLdDFy76QHXcpIonAq8BPVbUq0PW0JRG5BChV1ZVNd3tpGmrXOwoYBTypqiOBGkKse8UbT7/xVKAf0AtIwOlyaC7UrvexnPTve7AFeiHQu8l2JlAcoFralIhE44T5i6r6mmd3yRf/BfP8WRqo+trA2cAUEdmJ05U2HueOvYvnv+QQmte7EChU1c8823NwAj6UrzXARGCHqpapagPwGnAWoX+9oeVre9L5FmyBvgLI9nwSHoPzIcrcANfkd56+42eATar6aJNDc4EbPN/fALzZ3rW1FVW9R1UzVbUvznV9X1W/BSwBrvI0C6lzBlDVvUCBiAz27JoAbCSEr7XHbuAMEYn3/L5/cd4hfb09Wrq2c4Fve0a7nAEc+KJrxmeqGlRfwDeALcA24BeBrqeNzvEcnP9qrQU+93x9A6dPeTGw1fNn10DX2kbn/3Vgnuf7/sB/gXzgFSA20PW1wfmeDuR6rvcbQEo4XGvg/4DNwHrgn0BsqF1v4CWczwgacO7Ab27p2uJ0uTzhybZ1OCOAjuv97NF/Y4wJEcHW5WKMMaYFFujGGBMiLNCNMSZEWKAbY0yIsEA3xpgQYYFujDEhwgLdGGNCxP8DX+CgLEisb10AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores1)\n",
    "plt.plot(test_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  ReLU + Sigmoid + Cross Entropy + L2 + Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion2 = nn.CrossEntropyLoss()\n",
    "optimizer2 = optim.Adam(net2.parameters(), lr = 1e-3, weight_decay=1e-2)\n",
    "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion2, optimizer2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAdrElEQVR4nO3dd3hc9Z3v8fd3inqzrGJbcpFt3DDYGNkYHHpITFng0iF3KYE4BHJJQnZzSTbPpmx2s3fvBRJIQkIogYQSipeYEp4AoXe54gLYuEm2bMuWVSxZ1b/7x4xt2UiWbI10Zs58Xs8zj2bOOdJ8Dsd85sxvzpxjzjlERCTxBbwOICIisaFCFxHxCRW6iIhPqNBFRHxChS4i4hMhr564oKDAjRkzxqunFxFJSAsXLtzunCvsbp5nhT5mzBgqKiq8enoRkYRkZht6mqchFxERn1Chi4j4hApdRMQnVOgiIj6hQhcR8QkVuoiIT6jQRUR8IuEKfUt9Cz95dgXtnXu8jiIiElcSrtCXVNbx4NvrueuV1V5HERGJKwlX6HOnDuPiGaX8+tU1LNyw0+s4IiJxI+EKHeDH509heG46tz6xhKbWDq/jiIjEhYQs9Oy0MHdePp2Ntc387PlVXscREYkLCVnoALPK8vn6KeN47IONvLi82us4IiKeS9hCB7j1rAlMK83ln59axsYdzV7HERHxVEIXekoowK+umoEBNz+6iNaOTq8jiYh4JqELHWBkfga3XzadjzbV8+8aTxeRJJbwhQ5w1pRivnZyGQ+/u4EFSzd7HUdExBO+KHSA782dRPnoIdz29DLWbGv0Oo6IyKDzTaGHg5Hx9IyUIDf+aZGOTxeRpOObQgcYlpvGXVccx9qaXdw2/yOcc15HEhEZNL4qdICTxhfw3S9N5Nmlm3n43R6vpSoi4ju+K3SAb5w6jjMmFfGz51eytLLO6zgiIoPCl4UeCBi3XzqNouw0bn50EfXN7V5HEhEZcL4sdIAhmSncfdVxbKlv4Z+eWqrxdBHxPd8WOsCMUUO47exJvLRyKw+8vd7rOCIiA8rXhQ5w/RfK+OLkYv7zr6tYvqne6zgiIgPG94VuZvzXJceSn5nCLY8t1vHpIuJbvi90gPzMFO68fDrrdjTxk2dXeB1HRGRAJEWhA5w0roCbThvHExVVPLdM53sREf9JmkIH+PYXJzB9ZB4/fGY52xpavI4jIhJTSVXo4WCA2y+bRkt7J//76WU6lFFEfCWpCh1gXGEWt82dxKuf1PD4h5VexxERiZmkK3SAq08cw0njhvKz51ZSWatL14mIPyRloQcCxv+9dBoBM773lIZeRMQfei10MxtpZq+a2SozW2Fm3+pmGTOzu8xsjZktM7MZAxM3dkry0vn+OZN5d+0Onqyo8jqOiEi/9WUPvQP4rnNuMjAbuNnMphy0zNnAUdHbPOCemKYcIFfMHMmssnx+9vxKtjXqqBcRSWy9Frpzrto5tyh6vxFYBZQctNgFwMMu4j0gz8yGxzxtjAUCxs8vOoaWjj38ZMFKr+OIiPTLYY2hm9kY4Djg/YNmlQBdDxmp4vOlj5nNM7MKM6uoqak5vKQDZFxhFrecMZ7nP6rmbyu2eB1HROSI9bnQzSwLeBr4tnOu4eDZ3fzK5z5pdM7d65wrd86VFxYWHl7SATTvlHFMGpbNjxesYHdbp9dxRESOSJ8K3czCRMr8Eefc/G4WqQJGdnlcCiTM9+tTQgF+esFUNte38JvX1ngdR0TkiPTlKBcD7gdWOefu6GGxBcDV0aNdZgP1zrnqGOYccLPK8rlw+gh+9/pa1m9v8jqOiMhh68se+hzgH4EzzGxJ9HaOmd1oZjdGl3kBWAusAX4P3DQwcQfW98+ZTDho/Ntz+oBURBJPqLcFnHNv0f0YeddlHHBzrEJ5pTgnjW998Sj+44WPeWXVVs6cXOx1JBGRPkvKb4oeynVzyhhflMW/PbeS9s49XscREekzFfpBwsEAPzhnEut3NPPo+xu9jiMi0mcq9G6cPrGIE8cO5ZevrKahpd3rOCIifaJC74aZ8YNzJlPb1MbvXv/M6zgiIn2iQu/BMaW5XDh9BPe9uY7q+t1exxER6ZUK/RD+6csTcQ5u/9unXkcREemVCv0QSodkcM1Jo5m/qIrPanZ5HUdE5JBU6L248dRxpIWD3PXKaq+jiIgckgq9F0OzUrnmpDEsWLqZ1VsbvY4jItIjFXofzDt5LBnhIL/QXrqIxDEVeh8MyUzhujllPL+smo+3HHzmYBGR+KBC76MbTi4jOzXEL17SXrqIxCcVeh/lZaRw3ZwxvLhiC2u2aSxdROKPCv0wXDunjLRwgN++vtbrKCIin6NCPwz5mSlcMXMUzyzexOY6fXtUROKLCv0w3XByGQ64/611XkcRETmACv0wlQ7J4PxpI3jsg43UNbd5HUdEZB8V+hH4+qljaW7r5OF3N3gdRURkHxX6EZg0LIczJhXx4NvraGnv9DqOiAigQj9iXzt5LDub2/nLkk1eRxERAVToR2z22HwmDcvmwbfXE7lGtoiIt1ToR8jMuG7OGD7e0sh7a2u9jiMiokLvjwumlzAkI8yDb+sQRhHxngq9H9LCQa6cNYqXV22lsrbZ6zgikuRU6P30P2ePxsx4+N31XkcRkSSnQu+nEXnpzJ06jMc/rKS5rcPrOCKSxFToMXDNiWNobOnguaXVXkcRkSSmQo+BmWOGML4oi0c+2Oh1FBFJYir0GDAzrpo1iqWVdSzfVO91HBFJUir0GLl4RimpoQCPai9dRDyiQo+R3Iww5x07gr8s3sSuVn04KiKDT4UeQ1edMIqmtk4WLNnsdRQRSUIq9BiaMSqPScOyefQDnVZXRAafCj2GzIyrThjF8k0NfFSlD0dFZHCp0GPsguklpIYCPLmw0usoIpJkVOgxlpseZu7UYTyzeJMufiEig0qFPgAuKx9JQ0sHf1u51esoIpJEVOgD4MSxQynJS+fJCg27iMjg6bXQzewBM9tmZst7mH+amdWb2ZLo7V9jHzOxBALGpeWlvLVmO1U7dVpdERkcfdlD/wMwt5dl3nTOTY/eftr/WInvkuNLAXh6oa45KiKDo9dCd869Aegaa4epdEgGc8YV8OTCSvbs0TVHRWTgxWoM/UQzW2pmfzWzo2P0NxPepeWlVO3czXvrdngdRUSSQCwKfREw2jk3DbgbeKanBc1snplVmFlFTU1NDJ46vn356GFkp4Y07CIig6Lfhe6ca3DO7YrefwEIm1lBD8ve65wrd86VFxYW9vep415aOMi5xw7nr8uradIJu0RkgPW70M1smJlZ9P6s6N/UGEPUxceX0tzWyYvLt3gdRUR8LtTbAmb2GHAaUGBmVcCPgDCAc+63wCXAN8ysA9gNXOGc06eAUeWjhzB6aAZPL6ri4uiRLyIiA6HXQnfOXdnL/F8Bv4pZIp8xMy46rpRfvPIpm+p2U5KX7nUkEfEpfVN0EFw0owTn4L8XVXkdRUR8TIU+CEbmZ3BCWT5PL9qERqNEZKCo0AfJxceXsm57E4s21nkdRUR8SoU+SM45Zjhp4QDzNewiIgNEhT5IslJDzD16GM8u3azzpIvIgFChD6KLZpTS0NLB3z/e5nUUEfEhFfogmjO+gOKcVA27iMiAUKEPomDAuPC4El77pIbtu1q9jiMiPqNCH2QXHVdKxx7HgiWbvY4iIj6jQh9kE4dlM7Ukh/mLNewiIrGlQvfAxTNKWb6pgU+2NHodRUR8RIXugfOnjSAUMH04KiIxpUL3wNCsVE6fVMT8xZvo6NzjdRwR8QkVukcuOb6UmsZW3ljt/ys3icjgUKF75PSJReRnpvDUQg27iEhsqNA9khIKcOH0El5euY2dTW1exxERH1Che+iS40tp69zDs8t0TLqI9J8K3UNTRuQwZXiOhl1EJCZU6B675PhSllXV65h0Eek3FbrHLpg+gnDQeLKi0usoIpLgVOgeG5qVyllTipm/eBOtHTpPuogcORV6HLh85ihqm9p4aeVWr6OISAJToceBk8cXUJKXzp8/1LCLiBw5FXocCASMy8pH8ubq7VTWNnsdR0QSlAo9TlxaXkrA4Al9OCoiR0iFHidG5KVz6oRCnqio1Am7ROSIqNDjyOUzR7G1oZXXPtEJu0Tk8KnQ48iZk4sozE7lkfc3eB1FRBKQCj2OhIMBrpw1itc+rWHDjiav44hIglGhx5mvnDCKoBl/ek976SJyeFTocaY4J40vTx3Gnz+sZHebvjkqIn2nQo9DV88eTUNLBwuWbvI6iogkEBV6HJpVls+kYdk89M4GnHNexxGRBKFCj0Nmxj+eOJqV1Q0s2rjT6zgikiBU6HHqwuklZKeFeODt9V5HEZEEoUKPU5mpIa46YRR//aiajTt0fhcR6Z0KPY59dU4ZwYBx/1trvY4iIglAhR7HinPSuGB6CU9UVLGzqc3rOCIS51TocW7eKWPZ3d7JH/VFIxHpRa+FbmYPmNk2M1vew3wzs7vMbI2ZLTOzGbGPmbwmFGdz+sRCHnpnPS3t+qKRiPSsL3vofwDmHmL+2cBR0ds84J7+x5Ku5p0yjh1NbTy9qMrrKCISx3otdOfcG0DtIRa5AHjYRbwH5JnZ8FgFFJg9Np9pI/P47euf0a5zpYtID2Ixhl4CdL3MTlV02ueY2TwzqzCzipoanfO7r8yMb505nsra3czXXrqI9CAWhW7dTOv2++rOuXudc+XOufLCwsIYPHXyOH1iEdNKc7n772u0ly4i3YpFoVcBI7s8LgU2x+DvShdmxre/OIGqndpLF5HuxaLQFwBXR492mQ3UO+eqY/B35SCnTSzct5fe1qG9dBE5UF8OW3wMeBeYaGZVZna9md1oZjdGF3kBWAusAX4P3DRgaZOc9tJF5FBCvS3gnLuyl/kOuDlmieSQTptYyLSRefzyldVceFwJaeGg15FEJE7om6IJxsy4be4kqutbeODtdV7HEZE4okJPQCeOG8qZk4q459XP2LGr1es4IhInVOgJ6razJ9HU1sHdf1/jdRQRiRMq9AR1VHE2l88cxZ/e28D67U1exxGROKBCT2DfOesoUkIB/vOvH3sdRUTigAo9gRVlp3HTaeN4ccUW3vhUp1IQSXYq9AT3tVPGUlaQyY8WrKC1Q6fXFUlmKvQElxoK8pPzj2bd9ibufV2XqhNJZip0HzhlQiHnHjOcX726hspaXVBaJFmp0H3ih+dNJhgwfrRgBZEv74pIslGh+8Tw3HRuPWsCf/94G88s2eR1HBHxgArdR66bU0b56CH8eMFKtja0eB1HRAaZCt1HggHjvy45lpb2Tn4w/yMNvYgkGRW6z4wtzOKfvzyRVz7exvxFGnoRSSYqdB/aN/Ty7Aod9SKSRFToPhQMGHdcNh0c3PL4Yl2DVCRJqNB9atTQDH5+8TEs3ljHHS996nUcERkEKnQfO+/YEVw5axT3vPaZzvUikgRU6D73r+dNYUJxFrc+sYTq+t1exxGRAaRC97n0lCC/+coMWtr38PU/LqSlXSfwEvErFXoSGF+UzR2XTWNZVb2OTxfxMRV6kvjS0cO49awJzF+8ifvf0sWlRfxIhZ5Evnn6eOYePYz/eGEVL6/c6nUcEYkxFXoSCQSM2y+bxtSSXL752CIWb9zpdSQRiSEVepLJTA3xwLUzKc5J4/qHKlhbs8vrSCISIyr0JFSQlcpD183CgGse/EBnZhTxCRV6khpTkMn9186kdlcbV/7+PWoaW72OJCL9pEJPYtNH5vHgdbOormvhqt+/x45dKnWRRKZCT3KzyvJ54NqZVO5s5iv3va9SF0lgKnThxHFDue/qmazb3sSlv3uXzXU6RYBIIlKhCwBfOKqAP15/AjUNrVxyzzt8pqNfRBKOCl32mVWWz2PzZtPWuYfLfvsuSyrrvI4kIodBhS4HmFqSy5M3nkRGapDLf/cuzy7d7HUkEekjFbp8TllBJs/cNIdjSnL5X48t5pcvr9YJvUQSgApdujU0K5VHvnYCF80o4c6XP+XGPy2koaXd61gicggqdOlRaijI7ZdO41/OmczLq7Zx/t1vsXJzg9exRKQHKnQ5JDPja6eM5fF5s2lu6+R//OZt/vTeBg3BiMQhFbr0ycwx+Tx/y8nMKsvnh88s56t/+JBtjToHjEg8UaFLnxVmR07q9eN/mMI7n+3gy3e+wXPLNmtvXSRO9KnQzWyumX1iZmvM7LZu5l9rZjVmtiR6uyH2USUeBALGtXPKeP6WL1A6JINvPrqYGx6q0LdLReJAr4VuZkHg18DZwBTgSjOb0s2if3bOTY/e7otxTokz44uy+e+bTuKH507mnc92cNYdr3Pfm2tp69jjdTSRpNWXPfRZwBrn3FrnXBvwOHDBwMaSRBAKBrjh5LH87TunMKssn589v4q5v3yDVz/Z5nU0kaTUl0IvASq7PK6KTjvYxWa2zMyeMrOR3f0hM5tnZhVmVlFTU3MEcSUejczP4MHrZvHgtTPBwXUPfsjVD3zA8k31XkcTSSp9KXTrZtrBn4I9C4xxzh0LvAw81N0fcs7d65wrd86VFxYWHl5SiXunTyrixW+fwg/PncyyqjrOu/stbnpkIWu26URfIoOhL4VeBXTd4y4FDjjBh3Nuh3Nu74m0fw8cH5t4kmhSQpFhmDe+dzq3nDGe1z6p4aw7X+fmRxbpS0kiA6wvhf4hcJSZlZlZCnAFsKDrAmY2vMvD84FVsYsoiSgnLcytX5rIm987nW+cOo7XP63hnLve5Kt/+JB31mzXoY4iA8D68j+WmZ0D/AIIAg845/7dzH4KVDjnFpjZz4kUeQdQC3zDOffxof5meXm5q6io6PcKSGKob27noXfX89A769nR1MakYdl8dU4Z/zBtBOkpQa/jiSQMM1vonCvvdp5Xe0oq9OTU0t7JgiWbuf+tdXyytZHstBAXzyjlqhNGMaE42+t4InFPhS5xxznHB+tqeeT9jby4fAttnXs4piSXi2eUcP70EvIzU7yOKBKXVOgS13bsauWZJZt5emEVK6sbCAWMOeMLOO/Y4Xzp6GHkpoe9jigSN1TokjBWVTfwzJJNPL+smqqduwkHjZPGFfClo4s5a3IxRTlpXkcU8ZQKXRKOc44llXW88FE1f1u5lQ07mgGYWpLDaROKOG1iIdNH5hEK6vxyklxU6JLQnHOs3raLl1Zu5bVPtrFww072OMhODXHC2HzmjC9g9tihTCzOJhDo7ntwIv6hQhdfqW9u560123lrzXbe+Wz7vr333PQwM8cMYcboIcwYNYRppXk6JFJ851CFHhrsMCL9lZsR5txjh3PusZHvs1XtbOaDdbX7bi+vipwcLBgwJhZnc2xpLseW5jG1JIcJxdmkhVXy4k/aQxffqW1qY/HGnSzauJNlVfUsq6qnfnfkAtfBgDG+MItJw7OZOCybicXZTCjOpiQvXcM1khC0hy5JJT8zhTMnF3Pm5GIgMga/sbaZlZsbWFndwIrNDVSs38lfluw/JVFaOMDYgizGFWVRVpBJWUEGY4ZmMnpoJkMywpip7CX+qdDF98yM0dFyPvuY/acdamhp59Mtjazetos10dvSyjqeX7aZPV3euGanhhiZn8HI/HRKh2RQkpfOiLz06M808jNTVPgSF1TokrRy0sKUj8mnfEz+AdNbOzqprG1m/fZmNtQ2s3FHExtrm1lb08Qbn25nd3vnAcunBAMU56YyPCedopxUinPSKM5JpSg7jcLsVAqzUynISiUvPaxhHRlQKnSRg6SGgowvymZ80efPLeOco7apjer6FjbV7aa6bjfVDS1sqW+hur6FFZsbeGXVts+VPkTG74dmpjA0KzX6M4X8zBSGZqYwZO/PjMi0vIwU8jLChHWcvRwGFbrIYTCzSCFnpTK1JLfbZZxzNLZ2UNPYSk1jK9saW9mxq5Xtu1rZ3tjGjqY2djS1snFjM7VNbexq7ejx+bJTQ+RlhhmSkUJueuRnXkaYvPQwedFpeRmRW256mJz0MHnpKaSE9EKQjFToIjFmZuSkhclJCzOuMKvX5VvaO9nZ3EZtUxt1ze3saGqjrrmNnU3t7GyO3K/b3c7O5nYqa5up291O/e52DnWAWno4SG56eP8tI3zg4/QwOemh/ffT9r8g6LDOxKVCF/FYWjjI8Nx0huem9/l3Ovc4GlsixV7XHCn++t3tNEQf10dLf2/5V9Y2szw6v6nt88NBXaWGAvvKPVL2oQMe730ByOnyopCTFnnRyEoJ6XMCD6nQRRJQMGDRcfYURg89vN9t79xDw+79pV+/u52Glo59LwgNB0xvZ/uuNj6radr3+FDvDMwiw0Q5BxR/6IB3ADlpXeYftExGSlBHDPWDCl0kyYSDgX2fAxyuPXscu9o6Diz93R00tBz4YtD1BWLd9siLQWNLB829vDsIBozstBDZaaF9w1bZ0ReArvf3zw+RHZ2eHb2fzJ8fqNBFpM8Cgf2fD5QOOfzf3/vuoLEl8iKw9wVh7/BR1+mNLZEXjvU7mmhs6aCxpeOQHyDvlRYOREo+NVLyWWkhslPD++9H52VFXwSy9i6XGiYr+jgrNUQwAYeOVOgiMmj68+4AoKNzD02tnZF3BC37Sz9S+NGfrR0HPN7V2kFN467I/ej8vkgPB6MvBiEyoyWfGS3/zNRgZFrK/nkZe6elhshICe5bPjMlRFo4MChDSSp0EUkYoWCA3IwAuRlHfhWrvcNGu6Jlv7f8m1o7aWqNvENoau1kV+v+F4Sm1sjPTXW7993f1dpBW8eePj1nwCAzJVr6KSGuOmEUN5w89ojXoScqdBFJKl2HjfqrvXPPvoKPvAh00NzWse/Fobmtg6a2vfcjP5vaOik4wncovVGhi4gcoXAwsO9oo3iQvB8Hi4j4jApdRMQnVOgiIj6hQhcR8QkVuoiIT6jQRUR8QoUuIuITKnQREZ8wd6hzYQ7kE5vVABuO8NcLgO0xjJMoknG9k3GdITnXOxnXGQ5/vUc75wq7m+FZofeHmVU458q9zjHYknG9k3GdITnXOxnXGWK73hpyERHxCRW6iIhPJGqh3+t1AI8k43on4zpDcq53Mq4zxHC9E3IMXUREPi9R99BFROQgKnQREZ9IuEI3s7lm9omZrTGz27zOMxDMbKSZvWpmq8xshZl9Kzo938xeMrPV0Z9HcJne+GdmQTNbbGbPRR+Xmdn70fX+s5nFx9UEYsTM8szsKTP7OLrNT0yGbW1m34n++15uZo+ZWZoft7WZPWBm28xseZdp3W5fi7gr2m/LzGzG4TxXQhW6mQWBXwNnA1OAK81sirepBkQH8F3n3GRgNnBzdD1vA15xzh0FvBJ97EffAlZ1efx/gDuj670TuN6TVAPnl8CLzrlJwDQi6+7rbW1mJcAtQLlzbioQBK7An9v6D8Dcg6b1tH3PBo6K3uYB9xzOEyVUoQOzgDXOubXOuTbgceACjzPFnHOu2jm3KHq/kcj/4CVE1vWh6GIPARd6k3DgmFkpcC5wX/SxAWcAT0UX8dV6m1kOcApwP4Bzrs05V0cSbGsil8BMN7MQkAFU48Nt7Zx7A6g9aHJP2/cC4GEX8R6QZ2bD+/pciVboJUBll8dV0Wm+ZWZjgOOA94Fi51w1REofKPIu2YD5BfA9YO/l1IcCdc65juhjv23zsUAN8GB0mOk+M8vE59vaObcJ+H/ARiJFXg8sxN/buquetm+/Oi7RCt26mebb4y7NLAt4Gvi2c67B6zwDzczOA7Y55xZ2ndzNon7a5iFgBnCPc+44oAmfDa90JzpmfAFQBowAMokMNxzMT9u6L/r17z3RCr0KGNnlcSmw2aMsA8rMwkTK/BHn3Pzo5K17335Ff27zKt8AmQOcb2briQynnUFkjz0v+rYc/LfNq4Aq59z70cdPESl4v2/rLwLrnHM1zrl2YD5wEv7e1l31tH371XGJVugfAkdFPwlPIfIhygKPM8VcdNz4fmCVc+6OLrMWANdE718D/GWwsw0k59z3nXOlzrkxRLbt351zXwFeBS6JLuar9XbObQEqzWxidNKZwEp8vq2JDLXMNrOM6L/3vevt2219kJ627wLg6ujRLrOB+r1DM33inEuoG3AO8CnwGfAvXucZoHX8ApG3WcuAJdHbOUTGk18BVkd/5nuddQD/G5wGPBe9Pxb4AFgDPAmkep0vxus6HaiIbu9ngCHJsK2BnwAfA8uBPwKpftzWwGNEPidoJ7IHfn1P25fIkMuvo/32EZGjgPr8XPrqv4iITyTakIuIiPRAhS4i4hMqdBERn1Chi4j4hApdRMQnVOgiIj6hQhcR8Yn/D1p9sTU0kJKNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deZgV5Zn38e/dO/QG9AY0zSbNIhhQO7jvZgaMEaMmwawaE5NM1IxmkuhMJpM4M69JJhknueKbjG9i1CRq3DKiYeJMRAWNCigouzR70/QCDb0AvZ77/aMO2LQNfYBzOJxzfp/r6ouuOk9X3WXBz6efqnrK3B0REUl8afEuQEREokOBLiKSJBToIiJJQoEuIpIkFOgiIklCgS4ikiQGDHQze8DMGsxs5WE+NzP7qZlVm9k7ZnZG9MsUEZGBRNJDfxCYdYTPZwOV4a+bgZ8ff1kiInK0MgZq4O4LzWzsEZrMAR724Aml181siJmNcPcdR9pucXGxjx17pM2KiEhfb7755k53L+nvswEDPQLlwLZeyzXhdUcM9LFjx7J06dIo7F5EJHWY2ZbDfRaNi6LWz7p+5xMws5vNbKmZLW1sbIzCrkVE5IBoBHoNUNFreRRQ219Dd7/f3avcvaqkpN/fGERE5BhFI9DnAZ8N3+1yNtA80Pi5iIhE34Bj6Gb2KHAxUGxmNcA/AZkA7v4LYD5wBVAN7ANujFWxIiJyeJHc5XL9AJ878NWoVSQiIsdET4qKiCQJBbqISJKIxn3oIiJyGO1dPVQ3tFHX3E5Dawf1Le1cNqWUD4waEvV9KdBFRHrZ19lNQ0sQvG0d3Uf1s109TmNbB40t7dQ2t7OqtoX19a10hw59NKckP1uBLiKJZ39nD3s7jy4YW9u7WVXbzMrtLWxsbCMU01cfO63t3TS2dtDQ2nHUId4fMyjJy2byiAIumVTC1JGFlA8dRFlBNsV52WSmx2a0W4EuIkfUE3I279rLjj3t1Le009jWQXdP6H3t3KF5fxcNrR00tAbDC40tHbQeR0Bmphtji3JjFoAH5GanM2VEARdOzKa0IJuy/BxKC7IpyMnE+nsW/jDSzCjJz6YoN4uMGNfcHwW6iBzU3RNifUMbK7Y3s2p7Myu2N7NmRyv7u3oi+vmczDRK83Mozc9m8vB8LqwsobQgm7zsjH7nCDmc7Mx0Th1RQGVZHtkZ6cd2MClIgS6SAtyd2uZ2VtQ0s6q2mU079+LhYYyunhCNbR00tHTQ2NpBZ7j3nZuVztSRhcydWcHUkYVUDB1EaUEOJfnZZGf03/vMSDPsaLq0ElUKdJGTVEd3TxCw3UHAOtByYEijpT2iXnPT3q7wWHQzu/d1AZCeZlQMHXRwSCDdjOL8LGaOG0ZZQQ5TRuQzrbyQcUW5pKUpnBOJAl0kjtq7emho6aCupZ21dS3hHnQLtc372RMO4OORkWZMGp7PX506nGnlBUwrL2TKiAJyMjWMkYwU6CLHyN3p6H7v4mBnT4iGluCC4J59XQeHNLpDIRpbg+GM+pb28EXD4PvW9kMvGBblZjG1vJAzxwylND+bkvxsBmW9F7552RmUFQRj1LnZA//zzcpIi/kFRTl5KNBFegmFnC1N+1i5vZldbR0H1+/t7KGhVxg3tLbT0NJxSKAPJCs9jZL8bMoKsplQkse5pxRRFh6TLivIobI0jxGFORqDlmOmQJeUt2XXXhasbWDB2gaWb91z2Nvs8nMyKM3PpjQ/hzNGD6WsIIfCQe/d1paZFgR2aX42Q3OzSA+PP6eZUZyXFW6rsJbYUaBL0nN3lmzezcbGtoO96/qW8LBHczt1Le0AnFKSy5zTR3JaeSHTygsZUTjo4K12OZnphwx9iJyMFOiStNydP69p4KcvrGfF9uaD64cOzgzulS7I5pSSIqaNLOSyKaWMKcqNY7Uix0+BLknF3VlV28KCtQ3MX7GDtXWtjB42mB9cexrnV5ZQkpdN1mHuoRZJdAp0SXh7O7p5tXonC9Y28OK6BupbOjCD6aOG8OOPTWfOjJFxeQxb5ERToEtCcnde27iLxxZv40+r6ujsDpGfncEFE4u5dHIZF08qoTgvO95lipxQCnQ56e3v7OGRxVt5flUdofC0e/Wt7Wxr2k9+TgafqKpg9rThVI0dpuEUSWkKdDlp7d7byRNvbuP+hRvZ2dbJtPICCgdlAjCpLJ/bL5/IFaeN0FOPImEKdDlp7O/sYfWOZt7Y1MSCNQ28tXU3IYcLKou59dJKZo4bFu8SRU5qEQW6mc0CfgKkA7909+/3+XwM8ABQAjQBn3b3mijXKklofX0rv1+yjYXrG6lueO9FBtPKC7jlkgn81dThTCsvjG+RIgliwEA3s3TgPuBDQA2wxMzmufvqXs1+BDzs7g+Z2aXAPcBnYlGwJIc/r67n5y9v4M0tu8lMN849pZhZ00ZwWnkh00cVUlqQE+8SRRJOJD30mUC1u28EMLPHgDlA70A/Fbg9/P2LwH9Fs0hJHi3tXXxv3mqeequGsUWDuWv2ZK49c5TuSBGJgkgCvRzY1mu5BjirT5u3gWsJhmU+CuSbWZG774pKlZJwQiFn0669rKptoWV/MA1sZ3eIX72yibqWdm67dAK3XFqpu1JEoiiSQO9vNqG+r2z9O+BnZnYDsBDYDrxvhiMzuxm4GWD06NFHVaic3Lbu2sfSLU2s3N7CytpmVte29Puy3XHFuTz55XM4ffTQOFQpktwiCfQaoKLX8iigtncDd68FrgEwszzgWndvpg93vx+4H6Cqqiqm7/GWE2NVbTM/W1DNf6+sA4J3Sk4ZUcA1Z5QzrbyQqSMLKMl/bzilKDf74CyEIhJdkQT6EqDSzMYR9LznAp/s3cDMioEmdw8BdxHc8SJJamdbBy+ubaBm8TPUbt9CcUYa/zm1mOkVQyjJzyY9bRdMuBzyh8e+mO5OWPscdO6N7X5GfABGTI/tPlJNyw7Y8AIH3wSSSirOgpKJUd/sgIHu7t1mdgvwPMFtiw+4+yozuxtY6u7zgIuBe8zMCYZcvhr1SiVuek94tWBtA2/X7KGKtTyRfTdkhhttCH8dUDAKPjcPik6JXWHdHfDEDbBufuz2cUBaJnzsQZhyZez3lQp2bYAHr4TW2oHbJqMP/3tMAt08Tv93rKqq8qVLl8Zl3xKZfZ3d/Pb1LTz46mZqm9sxgw+MGsJlk0v5fPWt5LZuxD7/pyDsemuugd9/CtKz4HPPQfGE6BfX1Q6PfxbWPw+zfgCTPxz9fRzQ0wl/+BLULoNrfwVTr47dvlJB47vw0Ecg1A1zfwcF5fGu6MQbNASy84/pR83sTXev6u8zPSmaytpboGE1jD770NVdPTzw6iZ+uWgTTXs7OW9CEbd/aCIXTyoNxsM3LYRFrwVBOmz8+7c7pCII8oc+Ag9eAZd9Jwj3aHr7UdiwAK78D6i6Mbrb7s+nn4bfXQdPfh52bzo5QmjUB2HYuNjvxx02vgR7G49/Wz2d8OfvBd/f8EconXz825SD1ENPVfua4OE5UPcOXPqPcOHfAbCippk7Hl/O+oY2Lp5Uwq2XVnLmmF53pLjDr2fD7s1w23LIPMIDQA1r4eGroK0++vVbGnzkJ3DGZ6O/7cPpaIVHPgFbXj1x+zyS7EL4zNMwqt/OWnS4wwvfg1fujd4284bD556NyZBDKlAPXQ61d2cQ5jvXw/hLYME/09nVxS+4jp++sJ6ivCwe+vxMLppY8v6f3fgibH0NrvjRkcMcgt7XbcugtS76x5BdAHn91BdL2fnBbx67N53Y/fanowWeuBEevho+/RSM7vtoSBS4w/98G177GZx5I5x7a3S2mz8CsgZHZ1tyCAX6yaB2GZRNg/TMgdser7bGoNfctJE9V/+GZ1snMLb+W1yw6Pt49zqumHYb/3z1aRQODtcSCgUh3tEaLL9yb3DBM9KecVZubC+MnmhpaSfP8dw4P7iw+Ntr4K//FXKGRHf7G1+CN38NM2+G2T8EveD6pKdAj7fV8+Dxz8DkK+G6X0NGlMeae2utxx++ilDTZn5a+i/c92gP3aG1jC/6G35cnMPXdv4BysbDoNOD9qEemHcbLP/todv5yE8hQ4/qx13ByCDUH/oIPPu12OzjrK/ArHsU5glCgR5PoRC8+H9g0NDgXurHPwsffyg2Ydmyg64HPkx383ZuaP8G6xvG8vnzR/GxM0dRWZYPoYvhj7fDoh8Hdx9c+h145qvwzmNw4Tdg6jXBdtIzoSgGd63IsckfDl9+JbgNMNoyc/q/6C0nLQV6PK3+AzSugesegP274Y9fh8c+BR+8Kaq78Z4uWp/7Nul76/kbu4trr7mWq08vP3QelbQ0+PC9kJYBr/4E1v037HwXLvk2XPSNqNYjUZaRDWWnxrsKOQko0OMl1AMvfR9KpsCpHw0CNS0j+NW5+n+juisDzAfxo9J7uOdT1zNyyKD+G6alBRc70zLgjV/AZf8EF9wR1VpEJHYU6PGy8qmgB/yxh4IgBTjzhuCuk/1Nx7Xp1vZuHl28ledX1dPR3cPYolwuP/sM/vHc6aQNNI+KGcz6Ppx/+4l5dF9EokaBfqKEemDLX4IHKyDonZdNgylXHdpu6Jjg61h2EXIeX7qNHz6/jj37crjmjPP49NljmD6qEDuai1pmCnORBKRAP1GWPwLzbjl03dxH3uudH4fdezt5etl2Hlu8lfUNbVSNGcp3r5qpV7eJpBgF+omy8sngjoGrfxEsZw2G4acd1yZb2ru4Z/5annqzhs6eEDMqhvCTuTO4avrIo+uRi0hSUKCfCG2NwfwnF3w9ak/0/aV6J3/3xNvUt3Zw/cwKPnXWGKaMKIjKtkUkMSnQT4Q1z4CHYOpHj3tT79a38uBfNvPIG1sZr7f/iEgvCvQTYeUfoHgSlB7bvcJdPSHmLa/ld29s4a2te8hMN244dyzfmjWZQVnpUS5WRBKVAj3WWuuC2fku+tZRPz7d2R3iqbdquO/Famp272d8SS7/cMUUrjmjnKI8PXovIodSoMfa6mcAh2nXRNR8f2cPr1bvZMG6Bv68up6G1g6mjyrk7jlTuWRSqS52ishhKdBjbeXTUDoVSiYN2PSNjbv4wsNLaW3vJjcrnfMri7l+5mgumliiIBeRASnQY6m5Bra9HsyHMoC/bNjJTQ8upXzoIP7vp05l5rhhZGdofFxEIqdAj5WeLvjTXcH3Awy3vLJ+J194eAmjhw3md184O3jNm4jIUVKgx0J3Jzx5YzAl7l/9y2FfiNDe1cMvF23kpwuqGV+cy+++cJYudorIMVOgR1t3JzxxA6z7YzDJ1dlf6bfZC2vqufu51WzZtY/Z04bzrx89jWG5MXy5hYgkvYgmEjGzWWa2zsyqzezOfj4fbWYvmtkyM3vHzK6IfqkJ4rWfBWF+xY/6DfNQyPmX51Zz00NLyUxP47c3ncXPP32mwlxEjtuAPXQzSwfuAz4E1ABLzGyeu6/u1ezbwOPu/nMzOxWYD4yNQb0nv+oXYMR0mPnF933U0d3D1x9/m+fe2cHnzhnDt688lcz045+cS0QEIuuhzwSq3X2ju3cCjwFz+rRx4MBEIoVAbfRKTCBd+6FmMYy94H0fNe3t5LO/Wsxz7+zgrtmT+e5VUxXmIhJVkYyhlwPbei3XAH1nmPou8D9mdiuQC1weleoSzbbFwXzn4y48ZPWCtfV866kV7NnXyU/mzmDOjPI4FSgiySySLmJ/T7R4n+XrgQfdfRRwBfAbM3vfts3sZjNbamZLGxsbj77ak93mRWDpMPocIHjq866nV/D5B5dSlJvFM189X2EuIjETSQ+9BqjotTyK9w+p3ATMAnD318wsBygGGno3cvf7gfsBqqqq+v5PIfFtWgQjZ0BOAe7O7b9fzvOr6/jSReO540MT9aCQiMRUJD30JUClmY0zsyxgLjCvT5utwGUAZjYFyAGSsAt+BJ17YfubB8fP71+4kT+tquPvZ0/hrtlTFOYiEnMDBrq7dwO3AM8DawjuZlllZneb2YEXYn4d+KKZvQ08Ctzg7snXAz+Sra9DqAvGXcBfqnfygz+t5YrThvOFC8bFuzIRSRERPVjk7vMJbkXsve47vb5fDZwX3dISzOZFkJZBXeEMbv3FMsYV5/LD66ZrUi0ROWH0pGi0bFqEl5/JN5/dyP6uHn7/mbPJy9Z/XhE5cXQjdDR0tELtMt4dNIOF7zbyzb+exITS/HhXJSIpRoEeDVteA+/h3uoyplcM4TPnjI13RSKSgjQmEA2bF9JtGbzSPp4nrjmN9DSNm4vIiadAj4Lm1QtY2zOBz144hSkjCgb+ARGRGNCQy3Go2b2POx56mbzdq1k/aAa3XVYZ75JEJIUp0I/RM8u3c/m/v8z+6oWkm/Pxj32SnEw9PCQi8aNAPwb7Orv53rOrmTS8gH+raoH0bLLG9J2vTETkxFKgH4NHF2+jaW8n//jhKeTVvgYVMyEzJ95liUiKU6AfpY7uHu5fuIGzxw+jqhSoX/G+6XJFROJBgX6UnnyzhvqWDm69tBI2vxKs7OeFFiIiJ5oC/Sh09YT4+UsbOH30EM49pSiYvyVzMJSfGe/SREQU6EfjmeW11Ozez62XTggm3dq0CCrOggy94FlE4k+BfhQe/MsmJg/P55JJpdDWCI1rYJyGW0Tk5KBAj1B1Qxsrt7fwsaqKoHe+5cD4uS6IisjJQYEeoXlv12IGV35gRLCi+s+QlRe8ck5E5CSgQI+AuzNv+XbOGV9EWUEO7NkG7zwO066F9Mx4lyciAijQI7JiezObd+3jqukjgxWLfgzucOE34luYiEgvCvQIPLO8lsx0Y/a0EbB7Cyz7DZz5ORhSEe/SREQOUqAPoCfkPPt2LRdPKqVwcCYs/DewdDj/jniXJiJyCAX6AN7YtIuG1o5guKVpIyx/BKpuhMLyeJcmInKIiALdzGaZ2TozqzazO/v5/F4zWx7+etfM9kS/1PiYt7yWwVnpXD6lDF7+t+Ai6Pm3x7ssEZH3GfCNRWaWDtwHfAioAZaY2Tx3X32gjbvf3qv9rcDpMaj1hNvX2c0f39nBX08dzqCWTfDOY3DWVyB/eLxLExF5n0h66DOBanff6O6dwGPAnCO0vx54NBrFxdv8FXW0dnTziQ9WwMs/gIwcOP9v412WiEi/Ign0cmBbr+Wa8Lr3MbMxwDhgwfGXFn+PL9nG2KLBnJXXCCuegJlfhLzSeJclItKvSAK9v1fY+2HazgWedPeefjdkdrOZLTWzpY2NjZHWGBcbGttYvLmJT3xwNPbyDyArF879WrzLEhE5rEgCvQbofcP1KKD2MG3ncoThFne/392r3L2qpKQk8irj4PEl28hIMz4+uhlW/QHO+hLkFsW7LBGRw4ok0JcAlWY2zsyyCEJ7Xt9GZjYJGAq8Ft0ST7zO7hBPvVXDZVNKKVpybzBnyzm3xLssEZEjGjDQ3b0buAV4HlgDPO7uq8zsbjO7qlfT64HH3P1wwzEJY8Haena2dfKFCW2wZh6c/RUYPCzeZYmIHNGAty0CuPt8YH6fdd/ps/zd6JUVX48t2cbwghyqNv8nZBfCOV+Nd0kiIgPSk6J9NLS0s/DdRv5mUgu2bj6cewsMGhLvskREBqRA72Pe27WEHK5t+Q3kDIGzvhzvkkREIqJA7+Ppt7bzsbId5G55Ac67DXIK4l2SiEhEFOi9rKtrZfWOFm7L/AMMLoKZX4p3SSIiEVOg9/L0shpy0noYtXsJTL8esvPiXZKISMQU6GE9IeeZZbXMHdOG9XRA+RnxLklE5Kgo0MPe2LiLupZ2PlrWEKwYmRQTRopIClGghz29bDt52RlMtY2QUwhDx8W7JBGRo6JAB9o6uvnvFTuYPW04GXXLg9659TcnmYjIyUuBDjyzfDt7O3v45JmlUL9awy0ikpBSPtDdnd++vpUpIwqYkVULoS4FuogkpJQP9OXb9rBmRwufOms0tmNZsFKBLiIJKOUD/XdvbCU3K52rTy+H2mXBA0WFFQP/oIjISSalA715XxfPvl3LnNPLycvOgFpdEBWRxJXSgf7UWzV0dIf45MzR0LkPGtZouEVEElbKBrq788jircyoGMK08kKoXwneo0AXkYSVsoG+tWkf1Q1tXHNGebCiVhdERSSxpWygv7GpCYCzx4df/Fy7DPLKIH9EHKsSETl2KRvoizc1MXRwJhNKwjMq1i7TBVERSWgpHegfHDuMtDSDjjZoXAcjZsS7LBGRY5aSgb6jeT9bm/Yxc9ywYEXdO4BrylwRSWgRBbqZzTKzdWZWbWZ3HqbNx81stZmtMrNHoltmdC0Oj5+fNa7X+Dmohy4iCS1joAZmlg7cB3wIqAGWmNk8d1/dq00lcBdwnrvvNrPSWBUcDYs3NZGXncGUEfnBitplUFAO+WXxLUxE5DhE0kOfCVS7+0Z37wQeA+b0afNF4D533w3g7g3RLTO6Fm9q4swxQ8lIDx/+gQuiIiIJLJJALwe29VquCa/rbSIw0cxeNbPXzWxWtAqMtl1tHaxvaHtv/Ly9GXZVw0gNt4hIYhtwyAXo7z4+72c7lcDFwChgkZlNc/c9h2zI7GbgZoDRo0cfdbHRsGTzbgDOOhDoO94O/lQPXUQSXCQ99Bqg9/SDo4Dafto84+5d7r4JWEcQ8Idw9/vdvcrdq0pKSo615uOyeFMT2RlpnDaqMFhx8IKoAl1EElskgb4EqDSzcWaWBcwF5vVp81/AJQBmVkwwBLMxmoVGy5LNTZw+egjZGenBitplMGQ05BbFtzARkeM0YKC7ezdwC/A8sAZ43N1XmdndZnZVuNnzwC4zWw28CHzD3XfFquhj1dLexaraZmaOHfbeSl0QFZEkEckYOu4+H5jfZ913en3vwB3hr5PWX6p3EnI4b0JxsGJfE+zeDGfeEM+yRESiIqWeFH353UbysjM4Y8zQYMWO5cGf6qGLSBJImUB3d15e18h5E4rI7H3/OcCI6fErTEQkSlIm0Ksb2qhtbufiSb0eYq1dBsPGw6Ch8StMRCRKUibQX1rXCMCFE8O3S4ZCsF0XREUkeaRMoL/8biOVpXmUDxkUhPkf74CWGjjlsniXJiISFSkR6Ps6u1m8qYmLJpYEYf7sbfDmr+H8O2DGJ+NdnohIVER022Kie33jLjp7Qlw8sQSevRWW/RYu/CZc8vd6Q5GIJI2UCPSX1zUyKDOdDxY0BWF+zi1w6T/EuywRkahKiSGXl95t5JxTishueCdYMf36+BYkIhIDSR/om3fuZcuufcH4ee0yyMiBksnxLktEJOqSPtAXre91u2LtMhj+AUhPiZEmEUkxSR/oC9fvpGLYIMYOzQ7mPtd95yKSpJI60Lt6Qry2YRcXVJZgu6qhay+UnxHvskREYiKpA33Z1j20dXRzYWXxe/O2qIcuIkkqqQN90fpG0tOMc04JB3pWHhRNiHdZIiIxkdSBvnD9TmZUDKFwUCbUvhXMqpiWHu+yRERiImkDfc++Tt6p2cMFlcXQ0wV1KzTcIiJJLWkD/ZXqnbjDBZUl0LgWutsV6CKS1JI20Be9u5OCnAymjyrUBVERSQlJGejuzqL1jZw3oZiM9LQg0LMLYei4eJcmIhIzSRnoGxr3UtvcHgy3QBDoI6dDWlIerogIkKSBvvDd4HH/CyqLobsD6lZquEVEkl5EgW5ms8xsnZlVm9md/Xx+g5k1mtny8NcXol9q5Batb2R8cS4VwwZD9Z8h1AWjPhjPkkREYm7AWarMLB24D/gQUAMsMbN57r66T9Pfu/stMajxqHR09/D6xiY+XjUqeDvRi/cEL4KeODvepYmIxFQkPfSZQLW7b3T3TuAxYE5syzp2b27ezf6unmD8fO1zUL8CLvqWZlgUkaQXSaCXA9t6LdeE1/V1rZm9Y2ZPmllFfxsys5vNbKmZLW1sbDyGcge2cP1OMtKMs8cPhZfugaJKmHZdTPYlInIyiSTQ+3vppvdZfhYY6+4fAP4MPNTfhtz9fnevcveqkpKSo6s0QovWN3LmmKHkVT8HDavh4jvVOxeRlBBJoNcAvXvco4Da3g3cfZe7d4QX/x9wZnTKOwqhHvas/F/K6l7iM0Vrgt55yWSY+tETXoqISDxE0nVdAlSa2ThgOzAX+GTvBmY2wt13hBevAtZEtcpIrP8fhjw5lweygJXhdR9/WJNxiUjKGDDQ3b3bzG4BngfSgQfcfZWZ3Q0sdfd5wG1mdhXQDTQBN8Sw5v41BDfd3Gh388ubLiA9axCUTjnhZYiIxEtEg8vuPh+Y32fdd3p9fxdwV3RLOzq+awO7GEJu5QWkj9JbiUQk9STNk6L7695lQ2h48DJoEZEUlDSBTtNGNoeGc2GlAl1EUlNyBHp7C4M7d7E3fwzDC3PiXY2ISFwkRaC37lgHwNAKXQQVkdSVFIG+fvXbAEycMj3OlYiIxE9SBHrD5uCWxUlTPhDnSkRE4ifhAz0Ucnp2rmd3RgkZOXnxLkdEJG4SPtBX1jYzoqeW7iF6vZyIpLaED/QX1zYy1urIHzkp3qWIiMRVwgf6kjUbGGZt5JRNjHcpIiJxldCBvqutg73hWxYpOiW+xYiIxFlCB/qrG3YxhrpgYZgCXURSW0IH+ramfYxLq8MxGKaLoiKS2hI60Bta2qnMqMeGVEBGdrzLERGJq4QO9LqWdk5Jr9dwi4gICR7o9c3tjArt0AVRERESPNC7WurJ9b3qoYuIkMCBHgo5uXu3BgvqoYuIJG6g79rbSYUfuGVxfHyLERE5CSRsoNe3tFNmTcFCQXl8ixEROQkkbKA3tLZTZrvpziqArMHxLkdEJO4iCnQzm2Vm68ys2szuPEK768zMzawqeiX2r665gzLbg+cNj/WuREQSwoCBbmbpwH3AbOBU4HozO7WfdvnAbcAb0S6yP8GQy27SC0eeiN2JiJz0IumhzwSq3X2ju3cCjwFz+mn3z8APgfYo1ndYDa3tjEjbQ1rBiBOxOxGRk14kgV4ObOu1XBNed5CZnQ5UuPtzUaztiBqa91PMbsjXkIuICEQW6NbPOj/4oVkacC/w9QE3ZHazmS01s6WNjY2RV9mPfXsayKAH8tVDFxGByAK9BqjotTwKqO21nA9MA14ys83A2cC8/i6Muvv97l7l7lOiOuoAAAaZSURBVFUlJSXHXjVgbTvCe1cPXUQEIgv0JUClmY0zsyxgLjDvwIfu3uzuxe4+1t3HAq8DV7n70phUDHT1hMja3xAsqIcuIgJEEOju3g3cAjwPrAEed/dVZna3mV0V6wL709Aa3LIIqIcuIhKWEUkjd58PzO+z7juHaXvx8Zd1ZPUt7ZSxO1jIK4v17kREEkJCPinaEL4HvTtnmF5sISISlpCBXt/SQanplkURkd4SMtDrWtoZbrtJL9BToiIiByRkoNe3BE+JWoF66CIiByRkoO9s3sdQmnXLoohILwkZ6J3NdaQT0hi6iEgvCRno3hZ+U5F66CIiByVcoO/r7Ca/c2ewoB66iMhBCRfoDS0dlFn4oSL10EVEDkq4QK9raafUduMY5JbGuxwRkZNGwgX6gcf+ewaXQHpEMxeIiKSEhAv0hpYOSm0PpvFzEZFDJFygn19ZzIwh+0kr1Pi5iEhvCRfoU0YUMCzUhOmCqIjIIRIu0Onpgr2NusNFRKSPxAv0tvrgT42hi4gcIvECvVVPiYqI9CcBA10vhxYR6U8CBrp66CIi/Um8QC8YCZOvhMHF8a5EROSkkniPWk7+cPAlIiKHiKiHbmazzGydmVWb2Z39fP5lM1thZsvN7BUzOzX6pYqIyJEMGOhmlg7cB8wGTgWu7yewH3H309x9BvBD4N+jXqmIiBxRJD30mUC1u290907gMWBO7wbu3tJrMRfw6JUoIiKRiGQMvRzY1mu5BjirbyMz+ypwB5AFXBqV6kREJGKR9NCtn3Xv64G7+33ufgrwLeDb/W7I7GYzW2pmSxsbG4+uUhEROaJIAr0GqOi1PAqoPUL7x4Cr+/vA3e939yp3ryopKYm8ShERGVAkgb4EqDSzcWaWBcwF5vVuYGaVvRY/DKyPXokiIhKJAcfQ3b3bzG4BngfSgQfcfZWZ3Q0sdfd5wC1mdjnQBewGPhfLokVE5P3MPT43pJhZI7DlGH+8GNgZxXISRSoedyoeM6TmcafiMcPRH/cYd+93zDpugX48zGypu1fFu44TLRWPOxWPGVLzuFPxmCG6x514c7mIiEi/FOgiIkkiUQP9/ngXECepeNypeMyQmsediscMUTzuhBxDFxGR90vUHrqIiPSRcIE+0FS+ycDMKszsRTNbY2arzOxr4fXDzOx/zWx9+M+h8a412sws3cyWmdlz4eVxZvZG+Jh/H364LamY2RAze9LM1obP+Tkpcq5vD//9Xmlmj5pZTrKdbzN7wMwazGxlr3X9nlsL/DScbe+Y2RlHu7+ECvQIp/JNBt3A1919CnA28NXwcd4JvODulcAL4eVk8zVgTa/lHwD3ho95N3BTXKqKrZ8Af3L3ycB0guNP6nNtZuXAbUCVu08jeGhxLsl3vh8EZvVZd7hzOxuoDH/dDPz8aHeWUIFOBFP5JgN33+Hub4W/byX4B15OcKwPhZs9xGHmzElUZjaKYOqIX4aXjWDmzifDTZLxmAuAC4FfAbh7p7vvIcnPdVgGMMjMMoDBwA6S7Hy7+0Kgqc/qw53bOcDDHngdGGJmR/Xy5EQL9P6m8i2PUy0nhJmNBU4H3gDK3H0HBKEPlMavspj4D+CbQCi8XATscffu8HIynu/xQCPw6/BQ0y/NLJckP9fuvh34EbCVIMibgTdJ/vMNhz+3x51viRboEU3lmyzMLA94CvjbPi8RSTpmdiXQ4O5v9l7dT9NkO98ZwBnAz939dGAvSTa80p/wuPEcYBwwkuDFOLP7aZps5/tIjvvve6IF+tFO5ZuwzCyTIMx/5+5Ph1fXH/gVLPxnQ7zqi4HzgKvMbDPBUNqlBD32IeFfySE5z3cNUOPub4SXnyQI+GQ+1wCXA5vcvdHdu4CngXNJ/vMNhz+3x51viRboA07lmwzCY8e/Ata4e+/3s87jvZksPwc8c6JrixV3v8vdR7n7WILzusDdPwW8CFwXbpZUxwzg7nXANjObFF51GbCaJD7XYVuBs81scPjv+4HjTurzHXa4czsP+Gz4bpezgeYDQzMRc/eE+gKuAN4FNgD/EO96YnSM5xP8qvUOsDz8dQXBmPILBPPNvwAMi3etMTr+i4Hnwt+PBxYD1cATQHa864vB8c4AlobP938BQ1PhXAPfA9YCK4HfANnJdr6BRwmuEXQR9MBvOty5JRhyuS+cbSsI7gA6qv3pSVERkSSRaEMuIiJyGAp0EZEkoUAXEUkSCnQRkSShQBcRSRIKdBGRJKFAFxFJEgp0EZEk8f8Bs4shH3g1zWMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adam效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1f3/8dfJbkLYQlgDYTEskU2ICLiL+16Ximvdamu1at1bq7XWX+tSv/3qV0ul1t2CCmpRUbSKQEWUPexLSELCFrIRyEKSmfP7Y0YaQyIBZnIz976fj0cekztzmfu53snbO+eec66x1iIiIpEvyukCREQkNBToIiIuoUAXEXEJBbqIiEso0EVEXCLGqQ136dLF9u3b16nNi4hEpMWLFxdba1Obes2xQO/bty+LFi1yavMiIhHJGJPf3GtqchERcQkFuoiISyjQRURcQoEuIuISCnQREZdQoIuIuIQCXUTEJRzrhy4RzlpY+jqUF+z/mjEw+FzoMSL8dVSVwuJXoK46vNvpPgwyLwjvNrxm6zJYNzPwWfKaQWdBr9Ehf1sFuhyaJa/BB7cHF0yjFy385y9w9hMw+vpAwIfD1qXw9rVQvrmJGkIpGDhHXwPnPAWxR4RxWx5gLSx8ET75NfjrCO+xa6OSuyvQJcSshdpKiG93cP+uYht8+hCkHw8/+QCiGrXcVZXCuzfDh7+Czd/AyfeDiQ5d3QA5n8PHD0BSKtz0BaSF/o9jH78PvvwTzH0Kti2HC/4PjugUvu21VHIPiIlrnW1VlcLe3Yf/PtYHXz4O2W9Bxplw8Qtt47+lSxin7liUlZVlNfTfQdVl8N7PIecLOPtJGH1dy8+kp14FG/8Nt8yHlAFNr+P3BwLwyz+x7ww31PqfApf8A5JSwvP+ja2fBe/+FGp2tc72DqRTP7j89UBzULhYCwsmwWcPgb8+RG9q4JQH4YS79z8ZkAMyxiy21mY1+ZoC3YO2LYe3roGKrdB9aKDpYuRVcM6fIS7xh//t6n8FmjlOewSO/9WBt7VlMRStDUXV35fQHgadA1EhPvM/kF2FkDvX+Xbf+prA/zCry+C8v8DIK0O/jb27YcYvYdV7gf/Wg88Lzft2HQK9RoXmvTxIgd6W7dkJnz4IWTdAn7Hh396S1+GjuyGpC1z2auAPa84TgZ9uw+Dy16Bz//+uvy0bZv8RavcElrevgI594KezIVotdo7asxOmXQ9586BXVujb9svyoGILTPgdHHdH+K6FyEFRoLdl71wPq96FqBg4/Q8w9pbw/OHUVcPMewM9U/qfHGyq6PLf19d/GmhOsDbQrjnobFj6Jnx0F8QnQ0pGYL24RDjjscBZljjPVw/z/gyb5oT+vWPi4Pi7oP9JoX9vOWQK9LZq7UyYegWMvx1KN8HaDyHzwsDX21Cy/kA76PZsOOEeOOU3TTdVlOUHmlO2LYM+42HzfOh3IlzyErRrcvplEWllPxTo+s7slJpdgbPfrkfBqQ9BdCx89Qx8/vtAO3WoxXeAK6YGzryb0ykdbpgFH98HS14NnJ2d+tvWb6cWkUOiM3SnfHBnIDRv+vf3+6NWlsDeMPSiSEoNNJ20VM0uSOgQ+jpE5LDoDL0t2DQHpkwEX21g2V8P427bf3BBUkrrdcP7IQpzkUNireWf327m8Y/XUl3ra3KdRy8cypXH9gn5thXoreU/f4G4djDqmsDyEZ3gmJucrUlEQqq61seD76/g3SVbGD8ghaP7dGxyvSE9DuLb8kFQoLeGorWwaXagPfrEe52uRiTkfH7Lq/Pz2FC0x+lSHLU4v5QNRXu4Y0IGt0/IIDqqdbt6KtBbw7eTITo+MK+JiMuUVtZyx9SlzNtQTJd2cRgP91dPjo/hpeuO4ZRBXR3ZvgI93KrLYfkUGHbZ9/t9i0SQwrIqdu7eu9/zJXtqefhfKymurOWJS4Zx+TGhbxeWllOgh9vSN6CuCo692elKRA7JwrxSJk5egM/fdI+4tE5H8O4t4xnaSxfSnaZADye/L9Dc0md868wNLhJiNXU+7p+eTff2CTx20dD9Zro1wOj0TiQnxDpSn3yfAj2cvvkblOfD6Y86XYnIPm8vKuCl/+RSHzzjTk6I4bfnDmF0euf91n3ui41s2lnJazeM4cSBGi3c1mnuynCoq4b3b4VZv4EjTwvdLHUih6GmzscD07O5b1o2sdFRDOqWzKBuyRRV7OXyFxbw8le5NBxouHprBX+bk8Mlo9IU5hFCZ+ihtmsLTLk8MCvhSfcHfjR0XhxWUFrFLW8uZuWWCm49ZQB3nT5oX5e6XdV13P32Mn7/wWq+2VTKqPRA3+l3l2yhY2IsD52nidgiRYsC3RhzFvAMEA28aK19vNHrfYBXgY7BdR6w1s4Mca2RYe6TULwBrnwHBp7hdDUizF5bxJ1vLcNvLX+/NovTM7t97/UOR8Qy+ZosJs3J4S+freeTVdsBiIuJ4tmJR9MxsZXuiiSH7YCBboyJBp4HTgcKgYXGmBnW2tUNVvst8La1dpIxJhOYCfQNQ71tm98Haz8KTIClMBeH+fyWZ/69nme/2MiQHu3529WjSE9JanLdqCjDraccyU0n9KPeF2h2iY4yJMTq22UkackZ+hhgo7V2E4AxZipwIdAw0C3QPvh7B2BrKIuMGJsXQOVOGKK7w4uzGg72uXR0Go9dNLRF4RwfE028GmIjVksOXS+goMFyIXBso3UeAT41xvwSSAJOa+qNjDE3AzcD9OnjwgEIaz4IjAjN0Nm5OGdZQTm/eGMxxXtq+dPFw5h4TG9Pj970kpYEelOfhMYjDK4AXrHWPm2MGQe8bowZaq31f+8fWTsZmAyB6XMPpeA2y9pAoB85AeLbOV2NeIS1llfm5/HVxpJ9y/M2FJOaHM+0W8YxPK3pyaHEnVoS6IVA7wbLaezfpHIjcBaAtfZrY0wC0AUoCkWREWHLEqgoDEzAJdIKKmrquPvt5Xy2egf9U5NIiAk0qZw9rDu/v+AoXcz0oJYE+kIgwxjTD9gCTAQa32J8MzABeMUYMwRIAHaGstA2b82MwH1BB53ldCXiUjk795ATnM1wb72fpz9dR2FZNQ+fl8n1x/VVs4ocONCttfXGmNuAWQS6JL5krV1ljHkUWGStnQHcDfzdGPMrAs0x11mnboXkBGsDgd7vxMA85yIhZG1gatrHPlqzb3QnQLf28Uy9eSxZffcf4Sne1KLr2cE+5TMbPfdwg99XA8eFtrQIUrQ6cJPn8bc7XYm4TFVtPQ9MX8GM5Vs5bUhXbp+QQVTwTLxvlyTaqUuKNKBPQyisngEYGHyu05WIi+yt93H5CwtYtXUX9545iFtOGkBUK98wQSKLAj0U1syA9PHQzplJ7cWdnp+dw4otu5h01SjOHtbD6XIkAmhyrsNVvDHQ5KLBRBJC67bvZtKXG7loZE+FubSYAv1wrZkReByiGRUlNHx+y33Ts0lOiOXh849yuhyJIAr0w7VmBvQaDR3SnK5EXOLlr3JZXlDO787PpHOS+pJLyynQD0d5AWxdquYWCQlrLVO+3cyTn6zj1MFduWBET6dLkgiji6KHY80Hgcch5ztbh0S8mjofD72/kncWF3JCRheevmyEBgrJQVOgH441H0C3oZAywOlKJILll1Ty8zeWsGZbBbdPyOCOCRn7bj4hcjAU6Idq9w7Y/DWc/IDTlUgE+/fqHfzq7WVEGcPL1x3DKYPV9VUOnQL9UK39ELBqP5dDYq3l6U/X89zsjQzt1Z5JV42md+dEp8uSCKdAPxR11fD185A6GLrqfoty8N5eVMBzszfy46w0Hr2wZTefEDkQBfqhmPMElObANe+DLlzJQSqqqOGxj9Ywpl9nHr94uIbzS8io2+LB2rYcvnoWRl4NA05xuhqJQA//axV76/08fvEwhbmElAL9YPjq4V+3QWIKnPmY09VIBPp4xTY+WbWdX502kP6purOVhJaaXA7G1/8H27Phx69p3nNp1qqtu3h1fh51vv1vCTB3/U6O6tmen57Qz4HKxO0U6C1VkgNfPg6Dz4PMC52uRtqotxcV8ND7K4mLjqJTE8P2u3dI4M+XjSAmWl+OJfQU6C3h98OM2yE6Hs75s9PVSCsrr6pl9daKA643Y/lWpi4s4LgjU3h24tGktItvhepE/kuB3hJLXoX8/8D5z0J7TWXqJQs2lXDbP5dSvGdvi9a/9ZQB3HX6II30FEco0A+kYit89jD0PQFGXet0NdJKrLX8fd4mnvhkHekpiTx16XAS4364r3hKu3iO7KoLneIcBfoPsRY+uht8dXDBs+pz7hEVNXXc9042n6zazjnDuvPEJcNJToh1uiyRA1Kg/5BV78G6mXD6H6Bzf6erkVawdnsFt7yxhM2lVTx4zhBuOqGfZj2UiKFAb05VKcy8F3qMhLG/cLoaCZM563eybHM5AFV19bw6P4/khFim/HQsY/p1drg6kYOjQG/OrN9ATTlc+z5E6z+TG2UXlnP9y9/ib9BdfPyAFP738pF0bZ/gXGEih0hJ1ZSN/4blU+CEu6H7MKerkTCo8/m5b1o2qcnxfHrnSSQnBP4UNBRfIpkCvSmzHoSUDDjxPqcrkTB5YU4Oa7fvZvI1o+mQqAue4g4artZYeQHsXAtZ10Osvna70caiPTz7+UbOHd6DM47q7nQ5IiGjQG8sb17gsd+JztYhYbGrqo6731nOEXHRPHL+UU6XIxJSanJpLHduYDbFrvpjd5uVW3Zxy5uL2b6rhmcnHk1qsobmi7so0BuyNhDofU+AKH15cZN3FhXw4PsrSUmK462fjWNUH82WKe6j1GqodBNUbFFzi8vk7NzDfdOzyUrvxIe/PF5hLq6lQG8od07gsd9JztYhIfXa/Dxio6J4RjMgissp0BvKnQvJPSFlgNOVSIhU1NQxbXEh543ooTZzcT0F+nf8fsidF2hu0dwdrvHOokIqa31cP153CBL3U6B/p2g1VBWr/dxFfH7Lq/PzyErvxLC0Dk6XIxJ2CvTv5M4NPCrQXePLdUVsLq3iuuP6Ol2KSKtQoH8nd25gityOvZ2uRELklfl5dG+fwJkaDSoeoUAH8NVD/leB/ucS8Xx+y1Oz1jJvQzHXjEsnVjdkFo/QwCKAbcthbwX0V3fFSFe8Zy+3T1nK/JwSrhjTm5tO0MVQ8Y4WBbox5izgGSAaeNFa+3gT6/wYeASwwHJr7ZUhrDO8vut/rjP0iJNbXMn907KpqKkDYHtFDdW1Pp66dDiXZan5TLzlgIFujIkGngdOBwqBhcaYGdba1Q3WyQB+DRxnrS0zxnQNV8FhkTsXumZCu8gq2+v8fsu97yxn3Y7djB+QAsDAbsn87KT+HNVTvVrEe1pyhj4G2Git3QRgjJkKXAisbrDOT4HnrbVlANbaolAXGjb1e2HzAhj9E6crkYP0xjf5LMov4+nLRnDJ6DSnyxFxXEuuFvUCChosFwafa2ggMNAY85UxZkGwiWY/xpibjTGLjDGLdu7ceWgVh1rhIqivVnfFCLOlvJonPl7LCRlduHhU44+jiDe15Ay9qWGTttFyDJABnAykAfOMMUOtteXf+0fWTgYmA2RlZTV+D2fkzgUTBenHOV2JHEDl3np8NvCxefC9FVjgjz8ahtHIXhGgZYFeCDS8upQGbG1inQXW2jog1xizjkDALwxJleGUOxd6jIQjOjpdiTSjps7HIzNWMXVhwfeef/i8THp3TnSoKpG2pyWBvhDIMMb0A7YAE4HGPVjeB64AXjHGdCHQBLMplIWGRW0lFC6Ecbc6XYk0o6C0ilveXMzKLRVcMzad9JRAgKcmx3Pe8J4OVyfSthww0K219caY24BZBLotvmStXWWMeRRYZK2dEXztDGPMasAH3GutLQln4SGxeQH469R+7iBrLdMWF/L5mqavo3+9qQRrLf/4SRYThnRr5epEIkuL+qFba2cCMxs993CD3y1wV/AncuTOhahY6DPW6Uo8qbrWx2/fX8n0JYWkdTqCpLj9P44jenfksQuH0idFTSsiB+LtkaK5cyHtGIhLcroSz9lcUsXP3ljM2u0V3DEhg9snZBAdpYubIofDm4Huq4cvHoWtS+DUh5yuxnNq6nz85OVvKa2s5aXrjuGUQRrQJRIK3gv03Ttg2g2Q/x/IuhHG/9Lpijznmc83kFtcyRs3HsvxGV2cLkfENbwV6LWV8PdToaoEfvQCjJjodEWes3LLLibP3cRlo9MU5iIh5q1Az58PFYUw8Z8w+Fynq/Gcep+fB97NplNiHL89N9PpckRcx1uBnjsHouOg/ylOV+JJf/0yh5VbKvjrVaPokBjrdDkiruOxQJ8LaWMgTl3gWlNNnY/ff7CaKd9u5rzhPTh7qO4gJBIO3gn0qlLYlg0n/9rpSjylsKyKX7y5hOzCXfz8pAHcc8ZAzb0iEibeCfT8rwCrUaGtaHdNHZe/sICK6jpeuGa07u0pEmbeCfTcuRCbCL1GO12JZzz5yTq27qpm2s/HMzq9k9PliLied+6emzsX+oyDmDinK/GEhXmlvL4gn+vG91WYi7QSbwT67h2wc62aW1pJTZ2P+6dn06vjEdxzxiCnyxHxDG80ueTNCzwq0FvFs59vYNPOSl67YQxJ8d74iIm0Bd74a8udAwkdoMcIpytxtdp6P3+cuYZX5udx6eg0ThyY6nRJIp7ikUCfC31PgKhopytxFWv/exfBHRV7ufWfS1icX8YNx/Xj1+cMdrAyEW9yf6BvXwFleXDsLU5X4iqfrd7B/dOzKa2s3fdcYlw0z115tO4kJOIQdwe63wcf3AmJKTDsMqercYV6n5//+Ww9f/0yh6G92nPtuHQAoozh3OE9GJDazuEKRbzL3YH+zQuwZRFc/CIkpThdTUTaWl7N3+bkULnXB0DOzj0sKyjnijF9+N35mSTEqhlLpK1wb6CX5cEXf4CMM2HYpU5XE5HmbdjJ7VOWUlXro0u7eADiY6J48tLh/Dirt8PViUhj7gl0vx+2LoX6msDynCfARMN5/wOaO2SfmjofK7bswu+3P7je15tKeObzDQzsmsykq0fRX00pIm2eewJ9w6cw5fLvP3fu09AhzZl62qCNRXv4+RuL2Vi0p0XrXziyJ3+6eBiJTdy8WUTaHvf8pW5ZDCYKrp4eODNP6AA9RzpdVZvxUfY27pu2nITYaJ6ZOJLUYBNKc5LiYxie1kEzI4pEEPcE+o6VkJIBA051upI258V5m3jsozWM6tORv141mu4dEpwuSUTCwD2Bvn0F9B7jdBVtzoYdu3nik7WckdmN564cRVyMN6bvEfEid/x1V5fBrgLoNtTpStoUn99y//Rs2sXH8MeLhynMRVzOHX/hO1YFHrsPc7aONub1r/NYsrmch8/P3NftUETcyx2Bvn1l4FGBvk9hWRVPzlrHyYNSuWhkL6fLEZFW4I5A37ECErtAu25OV9Im5BVXctOrizDAYxcNVU8VEY9wx0XR7Sug+1ANIAI+XbWdu99eTnS0YdLVo0nrlOh0SSLSSiI/0H31ULQWxvzU6UocVe/z8/Rn65n0ZQ7DenXgr1eNondnhbmIl0R+oJdsAN9eT7ef79y9l9unLOXrTSWaNEvEwyI/0D1yQXTp5jK2lFfv93xVrY+nP11HeVUdT106nMs0aZaIZ0V+oO9YAdFx0GWg05WERb3PzxOfrOXv83KbXSc9JZH3fjGGzJ7tW7EyEWlrIj/Qt6+A1EEQHet0JSFXVFHDbVOW8m1uKdeOS+fqsek0ddm3d+dENbGIiBsCfSUceZrTVYREYVkV1/7jW7ZXBKYArq33ExNt+N/LR3LR0epLLiI/LLIDfU8RVBYFuixGOGstv3lvJdsrarhyTB+MgagowyWj0hjYLdnp8kQkAkR2oG9fEXh0wQXR95ZuYe76nTxyfibXHdfP6XJEJAJF9kjR4vWBx9QhztZxmIr37OXRD1czqk9HrhnX1+lyRCRCRfYZelk+xCZBUhenKzkgay0bi/awt96/32vPz95I1V4fT1wynOgojXYVkUPTokA3xpwFPANEAy9aax9vZr1LgXeAY6y1i0JWZXPK86FTepsf8l9RU8c9by/n09U7ml3nrtMHkqG2chE5DAcMdGNMNPA8cDpQCCw0xsyw1q5utF4ycDvwTTgKbVJZHnRMb7XNHYo12yq45Y3FFJZVc88ZA5u8wNkuPoax/VMcqE5E3KQlZ+hjgI3W2k0AxpipwIXA6kbr/QF4ErgnpBU2x9pAk0u/E1tlc429+U0+S/LLefi8TDokxgZLsryxIJ+3FhXgD7as5OzcQ8fEWKbePJasvp0dqVVEvKElgd4LKGiwXAgc23AFY8zRQG9r7YfGmGYD3RhzM3AzQJ8+fQ6+2oaqSqCu0pEz9PU7dvPIjFXU+Szf5pUw6arR9E9N4oHpK5ixfCsj0jrQvWPgvp3DenXgnjMHkZqsG0yISHi1JNCbaqC2+140Jgr4C3Ddgd7IWjsZmAyQlZVlD7D6DyvLDzx2at1A9/kt903LJjkhlqcuHc5v31/JxZPm07NDAptLq7j3zEHcctIAonRxU0RaWUsCvRBoOONTGrC1wXIyMBT4Mngjhe7ADGPMBWG9MFqeF3hs5TP0V+bnsaygnGcmjmTCkG6M7N2RO99axuqtFbx2w7Ecn9H2e9yIiDu1JNAXAhnGmH7AFmAicOV3L1prdwH7UswY8yVwT9h7uThwhl5QWsWfZ63jlEGpXDCiJwAp7eJ57YYx1PstsdGR3a1fRCLbARPIWlsP3AbMAtYAb1trVxljHjXGXBDuAptVng9HdIb41unqV1Baxc2vLybKwGM/Gva927oZYxTmIuK4FvVDt9bOBGY2eu7hZtY9+fDLaoGy/FY7O5+9tog731qG31qeu2oUvToe0SrbFRE5GJE7UrQ8H7oPD/tmJs/N4Y8z15LZoz1/u3o0fVJ0WzcRaZsis53A74PygrCfoS/dXMbjH6/l7KHdefcX4xXmItKmRWag794G/rqw9nCprffzwPQVdE1O4MlLh+sGEiLS5kVmk0sr9HCZ9GUO63bs5sVrs0hOcN/dkETEfSLzDL38u0APz7zhG3bs5rnZGzh/RE9Oy+wWlm2IiIRaZAZ6WT5goENayN/a57fcPz2bpPgYfnd+ZsjfX0QkXCIz0MvzoX1PiAn9/CivfZ3Hks2BSbe6tNP8KyISOSIz0Mvyw3JBtLCsiqdmreOkgan8SDdlFpEIE5mBXh76QUXf3aQZ4P/9aOj3RoKKiESCyAv0+r1QsTXkZ+jf3aT5/rMGk9ZJ/c1FJPJEXqDvKgRsSM/Q/X7LH2euZXR6J64Z27bvgCQi0pzIC/SyvMBjCM/QV2+roHjPXq4Zm655zEUkYkVeoJeHflDR/JxiAMYN0H09RSRyRV6gx7eH3mMhuUfI3nJ+TgkDUpPo1j4hZO8pItLaIi/Qh10KN86CqNDMrVLn8/NtbinjB+hOQyIS2SIv0EMsu7Ccqlof49XcIiIRzvOBPn9jCQBj+yvQRSSyKdBzSsjs0Z5OSXFOlyIiclg8Heg1dT4Wby5Tc4uIuIKnA31Jfhm19X6OO1IXREUk8nk60OfnlBAdZTimX2enSxEROWyeDvSvcooZkdaBdvGReeMmEZGGPBvou2vqyC7cpdGhIuIang30hXml+PyW4zSgSERcwrOBPn9jCXExUYxK7+R0KSIiIeHdQM8pYXSfTiTEhmYKARERp3ky0Msqa1m9rUL9z0XEVTwZ6As2BYb7jz9SgS4i7uHJQJ+fU0JiXDTD0zo6XYqISMh4NNCLGdOvM7HRntx9EXEpzyXajooacnZWqv1cRFzHc4H+dU6w/Vz9z0XEZTwX6PNziulwRCyZPdo7XYqISEh5MNBLGNc/hago43QpIiIh5alAn7N+J4Vl1RyfoeYWEXEfzwR65d56fvPuCgakJnFZVprT5YiIhJxn5o19atY6tu6q5p2fjSM+RsP9RcR9PHGGvji/jFe/zuPaselk9dXNLETEnVoU6MaYs4wx64wxG40xDzTx+l3GmNXGmGxjzOfGmPTQl3poauv93D89mx7tE7j3rMFOlyMiEjYHDHRjTDTwPHA2kAlcYYzJbLTaUiDLWjscmAY8GepCD9VHK7aysWgPj1xwlO5MJCKu1pIz9DHARmvtJmttLTAVuLDhCtba2dbaquDiAqBNXHW01vLyV3kMSE3i9MxuTpcjIhJWLQn0XkBBg+XC4HPNuRH4+HCKCpUlm8vJLtzFdeP7Yoz6nYuIu7WkDaKpJLRNrmjM1UAWcFIzr98M3AzQp0+fFpZ46F6Zn0dyQgwXj2oTXxhERMKqJWfohUDvBstpwNbGKxljTgMeBC6w1u5t6o2stZOttVnW2qzU1NRDqbfFtu+q4eMV27g8qzdJajsXEQ9oSaAvBDKMMf2MMXHARGBGwxWMMUcDLxAI86LQl3nw3vwmH5+1XDuur9OliIi0igMGurW2HrgNmAWsAd621q4yxjxqjLkguNpTQDvgHWPMMmPMjGberlVU1/r45zebmTC4G31SEp0sRUSk1bSoLcJaOxOY2ei5hxv8flqI6zpkBaVV3PLmYkoqa7nphH5OlyMi0mpc1bg8e10Rd05dht9a/n5tFmP76yYWIuIdrgn05QXl3PDKQgZ3b8/frh5FekqS0yWJiLQq1wT6ks1lWAuvXn8MXdsnOF2OiEirc83kXHnFlSTHx5CaHO90KSIijnBNoOeWVJHeJVEjQkXEs1wT6HnFlfRVu7mIeJgrAr223k9hWRX9uijQRcS7XBHohWVV+C06QxcRT3NFoOeVVALQt4tGhYqId7kj0IsDU7HrDF1EvMwdgV5SSXJCDJ2T4pwuRUTEMa4I9NxgDxd1WRQRL3NFoOeVVNJXPVxExOMiPtBr6/1sKaumn6bJFRGPi/hAL/iuy6LO0EXE4yI+0POKA10WNbuiiHhd5Ad6SaDLokaJiojXRX6gF1fSPiGGTomxTpciIuKoyA/0YA8XdVkUEa+L+EDP1SyLIiJAhAf63nofW8ur1cNFRIQID/SC0mr8FvppUi4RkcgO9Fx1WRQR2SdiA93vt7w4bxPJCTEM7JbsdDkiIo6L2ECfurCAb3JL+e25Q2gXH9A2AXcAAASjSURBVON0OSIijovIQN++q4Y/zVzD+AEp/Dirt9PliIi0CREX6NZaHvrXSur8fv508TD1PxcRCYq4QJ+5Yjufrd7BXacP1MVQEZEGIi7Q2yXEcHpmN244rp/TpYiItCkRdzXxpIGpnDQw1ekyRETanIg7QxcRkaYp0EVEXEKBLiLiEgp0ERGXUKCLiLiEAl1ExCUU6CIiLqFAFxFxCWOtdWbDxuwE8g/xn3cBikNYTqTw4n57cZ/Bm/vtxX2Gg9/vdGttk6MrHQv0w2GMWWStzXK6jtbmxf324j6DN/fbi/sMod1vNbmIiLiEAl1ExCUiNdAnO12AQ7y4317cZ/DmfntxnyGE+x2RbegiIrK/SD1DFxGRRhToIiIuEXGBbow5yxizzhiz0RjzgNP1hIMxprcxZrYxZo0xZpUx5o7g852NMZ8ZYzYEHzs5XWuoGWOijTFLjTEfBpf7GWO+Ce7zW8aYOKdrDDVjTEdjzDRjzNrgMR/nkWP9q+Dne6UxZooxJsFtx9sY85IxpsgYs7LBc00eWxPwbDDbso0xow52exEV6MaYaOB54GwgE7jCGJPpbFVhUQ/cba0dAowFbg3u5wPA59baDODz4LLb3AGsabD8BPCX4D6XATc6UlV4PQN8Yq0dDIwgsP+uPtbGmF7A7UCWtXYoEA1MxH3H+xXgrEbPNXdszwYygj83A5MOdmMRFejAGGCjtXaTtbYWmApc6HBNIWet3WatXRL8fTeBP/BeBPb11eBqrwIXOVNheBhj0oBzgReDywY4FZgWXMWN+9weOBH4B4C1ttZaW47Lj3VQDHCEMSYGSAS24bLjba2dC5Q2erq5Y3sh8JoNWAB0NMb0OJjtRVqg9wIKGiwXBp9zLWNMX+Bo4Bugm7V2GwRCH+jqXGVh8b/AfYA/uJwClFtr64PLbjze/YGdwMvBpqYXjTFJuPxYW2u3AH8GNhMI8l3AYtx/vKH5Y3vY+RZpgW6aeM61/S6NMe2A6cCd1toKp+sJJ2PMeUCRtXZxw6ebWNVtxzsGGAVMstYeDVTisuaVpgTbjS8E+gE9gSQCTQ6Nue14/5DD/rxHWqAXAr0bLKcBWx2qJayMMbEEwvxNa+27wad3fPcVLPhY5FR9YXAccIExJo9AU9qpBM7YOwa/koM7j3chUGit/Sa4PI1AwLv5WAOcBuRaa3daa+uAd4HxuP94Q/PH9rDzLdICfSGQEbwSHkfgIsoMh2sKuWDb8T+ANdba/2nw0gzgJ8HffwL8q7VrCxdr7a+ttWnW2r4EjusX1tqrgNnApcHVXLXPANba7UCBMWZQ8KkJwGpcfKyDNgNjjTGJwc/7d/vt6uMd1NyxnQFcG+ztMhbY9V3TTItZayPqBzgHWA/kAA86XU+Y9vF4Al+1soFlwZ9zCLQpfw5sCD52drrWMO3/ycCHwd/7A98CG4F3gHin6wvD/o4EFgWP9/tAJy8ca+D3wFpgJfA6EO+24w1MIXCNoI7AGfiNzR1bAk0uzwezbQWBHkAHtT0N/RcRcYlIa3IREZFmKNBFRFxCgS4i4hIKdBERl1Cgi4i4hAJdRMQlFOgiIi7x/wHZaBTtsp05LgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores1)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adam的效果明显要好。只是我看不懂原理。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
